Skip to content

D
datamanage
Commit 8dc4e341 by admin
Options

修改数据的获取方式和查询方式

parent c8f3851d
Showing with 393 additions and 514 deletions
src/main/java/com/zhiwei/manage/dao/DataDao.java
......@@ -20,41 +20,5 @@ public interface DataDao {
public List<Data> findBytempName(Date startTime, Date endTime, String tempName);
public List<Data> findByMethod(Date startTime, Date endTime, String method, Object value);
public int findBySourceFromLibrary(Date startTime, Date endTime, String source);
public int findBySpId(Date startTime, Date endTime, String syId);
/**
* 根据数据来源来搜索数据
*
* @param startTime
* @param endTime
* @param source
* @return List
*/
public List<EasyNews> findByAccurateSource(String startTime, String endTime, String source);
public int findByAccurateType(Date startTime, Date endTime, String type);
/**
* 根据可视化模板的id去搜索数据
*
* @param startTime
* @param endTime
* @param SpId
* @return List
*/
public List<EasyNews> findByAccurateSpId(String startTime, String endTime, String SpId);
public List<String> findSourceByLikeParam(String param);
public List<String> findTmpNameByLikeParam(String param);
public List<String> findTypeByLikeParam(String param);
public long finCountByDayData(Date startTime, Date endTime, Template temp);
public long finCountByDayData(Date startTime, Date endTime, String pt);
}
src/main/java/com/zhiwei/manage/dao/DataDaoImpl.java
......@@ -56,8 +56,6 @@ public class DataDaoImpl implements DataDao {
SimpleDateFormat sdf2 = new SimpleDateFormat("yyyy-MM-dd HH");
@Resource(name = "mongoTemplate")
private MongoTemplate mongo;
@Resource(name = "newmongoTemplate")
private MongoTemplate mongo2;
@Override
public List<Data> findByPt(Date startTime, Date endTime, String pt) {
......@@ -143,157 +141,135 @@ public class DataDaoImpl implements DataDao {
mongo.insert(data, "DayData");
}
@Override
public List<EasyNews> findByAccurateSource(String startTime, String endTime, String source) {
Calendar c = Calendar.getInstance();
int month = c.get(Calendar.MONTH) + 1;
int year = c.get(Calendar.YEAR);
Criteria criteria = new Criteria();
criteria.and("source").regex(source);
criteria.and("time").gte(startTime).lt(endTime);
return mongo2.find(Query.query(criteria), EasyNews.class,
"net_media_" + year + (month > 9 ? "" + month : "0" + month));
}
@Override
public List<EasyNews> findByAccurateSpId(String startTime, String endTime, String SpId) {
Calendar c = Calendar.getInstance();
int month = c.get(Calendar.MONTH) + 1;
int year = c.get(Calendar.YEAR);
Criteria criteria = new Criteria();
criteria.and("spyderInfoId").is(SpId);
criteria.and("time").gte(startTime).lt(endTime);
return mongo2.find(Query.query(criteria), EasyNews.class,
"net_media_" + year + (month > 9 ? "" + month : "0" + month));
}
@Override
public long finCountByDayData(Date startTime, Date endTime, String pt) {
String end = "";
Calendar cal = Calendar.getInstance();
cal.setTime(startTime);
MongoCollection<Document> col = null;
if (pt.equals("知乎")) {
if (DbDepot.dataCol != null) {
if (DbDepot.dataCol.get(pt) != null && DbDepot.month == cal.get(Calendar.MONTH)) {
col = (MongoCollection<Document>) DbDepot.dataCol.get(pt);
}
}
int resultCount = 0;
if (col == null) {
MongoClient mongoClient = null;
ServerBean ser = mongo.findOne(
Query.query(Criteria.where("pt").is(pt).andOperator(Criteria.where("other").is("data"))),
ServerBean.class);
mongoClient = new MongoClient(new MongoClientURI(
"mongodb://sjzno:sjzno1q2w3e4r@" + ser.getDbHost() + ":" + ser.getDbPort() + "/admin"));
// 连接到数据库
MongoDatabase mongo = mongoClient.getDatabase(ser.getDbName());
for (int i = 1; i < 6; i++) {
col = mongo.getCollection(ser.getCollection() + i);
Bson query = new BasicDBObject("update_at",
new BasicDBObject("$gte", startTime).append("$lte", endTime));
resultCount += col.count(query);
}
}
return resultCount;
}
col = getcol(startTime, endTime, pt);
long count = 0;
if (pt.equals("微博")) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");
Bson query = new BasicDBObject("time",
new BasicDBObject("$gte", sdf.format(startTime)).append("$lte", sdf.format(endTime)));
count = col.count(query);
} else if (pt.equals("微信") || pt.equals("网媒") || pt.equals("论坛") || pt.equals("贴吧") || pt.equals("平媒")) {
Bson query = new BasicDBObject("time", new BasicDBObject("$gte", startTime).append("$lte", endTime))
.append("pt", pt);
count = col.count(query);
} else {
Bson query = new BasicDBObject("time", new BasicDBObject("$gte", startTime).append("$lte", endTime));
count = col.count(query);
}
return count;
}
// @Override
// public long finCountByDayData(Date startTime, Date endTime, String pt) {
// String end = "";
// Calendar cal = Calendar.getInstance();
// cal.setTime(startTime);
// MongoCollection<Document> col = null;
// if (pt.equals("知乎")) {
// if (DbDepot.dataCol != null) {
// if (DbDepot.dataCol.get(pt) != null && DbDepot.month == cal.get(Calendar.MONTH)) {
// col = (MongoCollection<Document>) DbDepot.dataCol.get(pt);
// }
// }
// int resultCount = 0;
// if (col == null) {
// MongoClient mongoClient = null;
// ServerBean ser = mongo.findOne(
// Query.query(Criteria.where("pt").is(pt).andOperator(Criteria.where("other").is("data"))),
// ServerBean.class);
// mongoClient = new MongoClient(new MongoClientURI(
// "mongodb://sjzno:sjzno1q2w3e4r@" + ser.getDbHost() + ":" + ser.getDbPort() + "/admin"));
// // 连接到数据库
// MongoDatabase mongo = mongoClient.getDatabase(ser.getDbName());
// for (int i = 1; i < 6; i++) {
// col = mongo.getCollection(ser.getCollection() + i);
// Bson query = new BasicDBObject("update_at",
// new BasicDBObject("$gte", startTime).append("$lte", endTime));
// resultCount += col.count(query);
// }
// }
// return resultCount;
// }
// col = getcol(startTime, endTime, pt);
// long count = 0;
// if (pt.equals("微博")) {
// SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");
// Bson query = new BasicDBObject("time",
// new BasicDBObject("$gte", sdf.format(startTime)).append("$lte", sdf.format(endTime)));
// count = col.count(query);
// } else if (pt.equals("微信") || pt.equals("网媒") || pt.equals("论坛") || pt.equals("贴吧") || pt.equals("平媒")) {
// Bson query = new BasicDBObject("time", new BasicDBObject("$gte", startTime).append("$lte", endTime))
// .append("pt", pt);
// count = col.count(query);
// } else {
// Bson query = new BasicDBObject("time", new BasicDBObject("$gte", startTime).append("$lte", endTime));
// count = col.count(query);
// }
// return count;
// }
@Override
public long finCountByDayData(Date startTime, Date endTime, Template temp) {
String end = "";
Calendar cal = Calendar.getInstance();
String pt = temp.getPt();
cal.setTime(startTime);
MongoCollection<Document> col = getcol(startTime, endTime, pt);
Bson query = new BasicDBObject("time", new BasicDBObject("$gte", startTime).append("$lte", endTime))
.append("spyderInfoId", temp.getSpyderInfoId());
return col.count(query);
}
// @Override
// public long finCountByDayData(Date startTime, Date endTime, Template temp) {
// String end = "";
// Calendar cal = Calendar.getInstance();
// String pt = temp.getPt();
// cal.setTime(startTime);
// MongoCollection<Document> col = getcol(startTime, endTime, pt);
// Bson query = new BasicDBObject("time", new BasicDBObject("$gte", startTime).append("$lte", endTime))
// .append("spyderInfoId", temp.getSpyderInfoId());
// return col.count(query);
// }
public MongoCollection<Document> getcol(Date startTime, Date endTime, String pt) {
String end = "";
Calendar cal = Calendar.getInstance();
cal.setTime(startTime);
if (DbDepot.dataCol != null) {
if (DbDepot.dataCol.get(pt) != null && DbDepot.month == cal.get(Calendar.MONTH)) {
return (MongoCollection<Document>) DbDepot.dataCol.get(pt);
} else {
ServerBean ser = mongo.findOne(
Query.query(Criteria.where("pt").is(pt).andOperator(Criteria.where("other").is("data"))),
ServerBean.class);
if (ser.isTable()) {
int month = cal.get(Calendar.MONTH) + 1;
end = cal.get(Calendar.YEAR) + "" + (month > 9 ? "" + month : "0" + month);
System.out.println(end);
}
MongoClient mongoClient = null;
if (ser.getPt().equals("微信全量")) {
mongoClient = new MongoClient(new MongoClientURI(
"mongodb://sjzzc:sjzzc1q2w3e4r@" + ser.getDbHost() + ":" + ser.getDbPort() + "/admin"));
} else {
mongoClient = new MongoClient(new MongoClientURI(
"mongodb://sjzno:sjzno1q2w3e4r@" + ser.getDbHost() + ":" + ser.getDbPort() + "/admin"));
}
// if (ser.getDbUsername() != null &&
// ser.getDbUsername().length() > 2) {
// ServerAddress sa = new ServerAddress(ser.getDbHost(),
// ser.getDbPort());
// List<MongoCredential> mongoCredentialList = new
// ArrayList<MongoCredential>();
// mongoCredentialList.add(MongoCredential.createMongoCRCredential(ser.getDbUsername(),
// ser.getDbName(), ser.getDbPassword().toCharArray()));
// new MongoClient(sa, mongoCredentialList);
// mongoClient = new MongoClient(sa, mongoCredentialList);
// } else
// mongoClient = new MongoClient(ser.getDbHost(),
// ser.getDbPort());
// 连接到数据库
MongoDatabase mongo = mongoClient.getDatabase(ser.getDbName());
DbDepot.dataCol.put(pt, mongo.getCollection(ser.getCollection() + end));
DbDepot.month = cal.get(Calendar.MONTH);
return mongo.getCollection(ser.getCollection() + end);
}
} else {
DbDepot.dataCol = new ConcurrentHashMap<String, Object>();
ServerBean ser = mongo.findOne(
Query.query(Criteria.where("pt").is(pt).andOperator(Criteria.where("other").is("data"))),
ServerBean.class);
if (ser.isTable()) {
int month = cal.get(Calendar.MONTH) + 1;
end = cal.get(Calendar.YEAR) + "" + (month > 9 ? "" + month : "0" + month);
}
MongoClient mongoClient = null;
if (ser.getDbUsername() != null) {
ServerAddress sa = new ServerAddress(ser.getDbHost(), ser.getDbPort());
List<MongoCredential> mongoCredentialList = new ArrayList<MongoCredential>();
mongoCredentialList.add(MongoCredential.createMongoCRCredential(ser.getDbUsername(), ser.getDbName(),
ser.getDbPassword().toCharArray()));
mongoClient = new MongoClient(sa, mongoCredentialList);
} else
mongoClient = new MongoClient(ser.getDbHost(), ser.getDbPort());
MongoDatabase mongo = mongoClient.getDatabase(ser.getDbName());
DbDepot.dataCol.put(pt, mongo.getCollection(ser.getCollection() + end));
return mongo.getCollection(ser.getCollection() + end);
}
}
// public MongoCollection<Document> getcol(Date startTime, Date endTime, String pt) {
// String end = "";
// Calendar cal = Calendar.getInstance();
// cal.setTime(startTime);
// if (DbDepot.dataCol != null) {
// if (DbDepot.dataCol.get(pt) != null && DbDepot.month == cal.get(Calendar.MONTH)) {
// return (MongoCollection<Document>) DbDepot.dataCol.get(pt);
// } else {
// ServerBean ser = mongo.findOne(
// Query.query(Criteria.where("pt").is(pt).andOperator(Criteria.where("other").is("data"))),
// ServerBean.class);
// if (ser.isTable()) {
// int month = cal.get(Calendar.MONTH) + 1;
// end = cal.get(Calendar.YEAR) + "" + (month > 9 ? "" + month : "0" + month);
// System.out.println(end);
// }
// MongoClient mongoClient = null;
// if (ser.getPt().equals("微信全量")) {
// mongoClient = new MongoClient(new MongoClientURI(
// "mongodb://sjzzc:sjzzc1q2w3e4r@" + ser.getDbHost() + ":" + ser.getDbPort() + "/admin"));
// } else {
// mongoClient = new MongoClient(new MongoClientURI(
// "mongodb://sjzno:sjzno1q2w3e4r@" + ser.getDbHost() + ":" + ser.getDbPort() + "/admin"));
// }
// // if (ser.getDbUsername() != null &&
// // ser.getDbUsername().length() > 2) {
// // ServerAddress sa = new ServerAddress(ser.getDbHost(),
// // ser.getDbPort());
// // List<MongoCredential> mongoCredentialList = new
// // ArrayList<MongoCredential>();
// // mongoCredentialList.add(MongoCredential.createMongoCRCredential(ser.getDbUsername(),
// // ser.getDbName(), ser.getDbPassword().toCharArray()));
// // new MongoClient(sa, mongoCredentialList);
// // mongoClient = new MongoClient(sa, mongoCredentialList);
// // } else
// // mongoClient = new MongoClient(ser.getDbHost(),
// // ser.getDbPort());
// // 连接到数据库
// MongoDatabase mongo = mongoClient.getDatabase(ser.getDbName());
// DbDepot.dataCol.put(pt, mongo.getCollection(ser.getCollection() + end));
// DbDepot.month = cal.get(Calendar.MONTH);
// return mongo.getCollection(ser.getCollection() + end);
// }
// } else {
// DbDepot.dataCol = new ConcurrentHashMap<String, Object>();
// ServerBean ser = mongo.findOne(
// Query.query(Criteria.where("pt").is(pt).andOperator(Criteria.where("other").is("data"))),
// ServerBean.class);
// if (ser.isTable()) {
// int month = cal.get(Calendar.MONTH) + 1;
// end = cal.get(Calendar.YEAR) + "" + (month > 9 ? "" + month : "0" + month);
// }
// MongoClient mongoClient = null;
// if (ser.getDbUsername() != null) {
// ServerAddress sa = new ServerAddress(ser.getDbHost(), ser.getDbPort());
// List<MongoCredential> mongoCredentialList = new ArrayList<MongoCredential>();
// mongoCredentialList.add(MongoCredential.createMongoCRCredential(ser.getDbUsername(), ser.getDbName(),
// ser.getDbPassword().toCharArray()));
// mongoClient = new MongoClient(sa, mongoCredentialList);
// } else
// mongoClient = new MongoClient(ser.getDbHost(), ser.getDbPort());
// MongoDatabase mongo = mongoClient.getDatabase(ser.getDbName());
// DbDepot.dataCol.put(pt, mongo.getCollection(ser.getCollection() + end));
// return mongo.getCollection(ser.getCollection() + end);
// }
// }
public static void main(String[] args) {
MongoClient mongoClient = new MongoClient(
......@@ -306,31 +282,6 @@ public class DataDaoImpl implements DataDao {
}
@Override
public int findBySourceFromLibrary(Date startTime, Date endTime, String source) {
Calendar c = Calendar.getInstance();
c.setTime(startTime);
int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1;
return (int) mongo2.count(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("type").is(source)),
"net_media_" + year + (month > 9 ? "" + month : "0" + month));
}
@SuppressWarnings("unchecked")
@Override
public List<String> findSourceByLikeParam(String param) {
Calendar c = Calendar.getInstance();
int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1;
try {
return mongo2.getCollection("net_media_" + year + (month > 9 ? "" + month : "0" + month)).distinct("source",
new BasicDBObject("source", new BasicDBObject("$regex", param)));
} catch (Exception e) {
return mongo2.getCollection("net_media_" + year + (month > 9 ? "" + month : "0" + month)).distinct("source",
new BasicDBObject("source", new BasicDBObject("$regex", param)));
}
}
@SuppressWarnings("unchecked")
@Override
public List<String> findTmpNameByLikeParam(String param) {
......@@ -343,92 +294,52 @@ public class DataDaoImpl implements DataDao {
}
}
@Override
public int findByAccurateType(Date startTime, Date endTime, String type) {
Calendar c = Calendar.getInstance();
c.setTime(startTime);
int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1;
return (int) mongo2.count(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("type").is(type)),
"net_media_" + year + (month > 9 ? "" + month : "0" + month));
}
@SuppressWarnings("unchecked")
@Override
public List<String> findTypeByLikeParam(String param) {
Calendar c = Calendar.getInstance();
int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1;
try {
DBObject obj = new BasicDBObject();
obj.put("pt", new BasicDBObject("$ne", "微信"));
obj.put("type", new BasicDBObject("$regex", param));
return mongo2.getCollection("net_media_" + year + (month > 9 ? "" + month : "0" + month)).distinct("type",
obj);
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
@Override
public List<Data> findByMethod(Date startTime, Date endTime, String method, Object value) {
List<Data> list = new ArrayList<Data>();
if (DbDepot.dbCons.get("data") != null) {
DBCollection coll = (DBCollection) DbDepot.dbCons.get("data");
DBObject obj = new BasicDBObject();
if (method != null) {
obj.put(method, value);
obj.put("time", new BasicDBObject("$gte", startTime).append("$lt", endTime));
} else {
obj.put("time", new BasicDBObject("$gte", startTime).append("$lt", endTime));
}
DBCursor cursor = coll.find(obj).sort(new BasicDBObject("time", -1));
while (cursor.hasNext()) {
Map<String, Object> map = (Map<String, Object>) cursor.next();
Data d = Change.toBean(Data.class, map);
list.add(d);
}
cursor.close();
} else {
System.out.println("user------------------");
ServerBean sb = mongo.findOne(Query.query(Criteria.where("other").is("data")), ServerBean.class);
DBCollection coll = null;
if (sb.getDbUsername() == null || sb.getDbUsername().equals("")) {
coll = new MongoConnect(sb.getDbHost(), sb.getDbPort(), sb.getDbName())
.getCollection(sb.getCollection());
} else {
coll = new MongoConnect(sb.getDbHost(), sb.getDbPort(), sb.getDbUsername(), sb.getDbPassword(),
sb.getDbName()).getCollection(sb.getCollection());
DbDepot.dbCons.put(sb.getOther(), coll);
}
DBObject obj = new BasicDBObject();
if (method != null) {
obj.put(method, value);
obj.put("time", new BasicDBObject("$gte", startTime).append("$lt", endTime));
} else {
obj.put("time", new BasicDBObject("$gte", startTime).append("$lt", endTime));
}
DBCursor cursor = coll.find(obj).sort(new BasicDBObject("time", -1));
while (cursor.hasNext()) {
Map<String, Object> map = (Map<String, Object>) cursor.next();
Data d = Change.toBean(Data.class, map);
list.add(d);
}
cursor.close();
}
return list;
}
@Override
public int findBySpId(Date startTime, Date endTime, String syId) {
Calendar c = Calendar.getInstance();
c.setTime(startTime);
int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1;
return (int) mongo2.count(
Query.query(Criteria.where("time").gt(startTime).lt(endTime).and("spyderInfoId").is(syId)),
EasyNews.class, "net_media_" + year + (month > 9 ? "" + month : "0" + month));
}
// @Override
// public List<Data> findByMethod(Date startTime, Date endTime, String method, Object value) {
// List<Data> list = new ArrayList<Data>();
// if (DbDepot.dbCons.get("data") != null) {
// DBCollection coll = (DBCollection) DbDepot.dbCons.get("data");
// DBObject obj = new BasicDBObject();
// if (method != null) {
// obj.put(method, value);
// obj.put("time", new BasicDBObject("$gte", startTime).append("$lt", endTime));
// } else {
// obj.put("time", new BasicDBObject("$gte", startTime).append("$lt", endTime));
// }
// DBCursor cursor = coll.find(obj).sort(new BasicDBObject("time", -1));
// while (cursor.hasNext()) {
// Map<String, Object> map = (Map<String, Object>) cursor.next();
// Data d = Change.toBean(Data.class, map);
// list.add(d);
// }
// cursor.close();
// } else {
// ServerBean sb = mongo.findOne(Query.query(Criteria.where("other").is("data")), ServerBean.class);
// DBCollection coll = null;
// if (sb.getDbUsername() == null || sb.getDbUsername().equals("")) {
// coll = new MongoConnect(sb.getDbHost(), sb.getDbPort(), sb.getDbName())
// .getCollection(sb.getCollection());
// } else {
// coll = new MongoConnect(sb.getDbHost(), sb.getDbPort(), sb.getDbUsername(), sb.getDbPassword(),
// sb.getDbName()).getCollection(sb.getCollection());
// DbDepot.dbCons.put(sb.getOther(), coll);
// }
// DBObject obj = new BasicDBObject();
// if (method != null) {
// obj.put(method, value);
// obj.put("time", new BasicDBObject("$gte", startTime).append("$lt", endTime));
// } else {
// obj.put("time", new BasicDBObject("$gte", startTime).append("$lt", endTime));
// }
// DBCursor cursor = coll.find(obj).sort(new BasicDBObject("time", -1));
// while (cursor.hasNext()) {
// Map<String, Object> map = (Map<String, Object>) cursor.next();
// Data d = Change.toBean(Data.class, map);
// list.add(d);
// }
// cursor.close();
// }
// return list;
// }
}
src/main/java/com/zhiwei/manage/dao/ServerDao.java
......@@ -34,5 +34,7 @@ public interface ServerDao {
public List<Channel> findDisChannels();
public List<String> findDisChan();
public ServerBean findServerBean(String pt);
}
src/main/java/com/zhiwei/manage/dao/ServerDaoImpl.java
......@@ -26,6 +26,7 @@ public class ServerDaoImpl implements ServerDao {
@Resource(name = "mongoTemplate")
private MongoTemplate mongo;
@Override
public void insert(ServerBean serverBean) {
mongo.insert(serverBean);
......@@ -74,33 +75,37 @@ public class ServerDaoImpl implements ServerDao {
@Override
public void findPt(String pt) {
// List<ServerBean> list = mongo.find(Query.query(Criteria.where("pt").is(pt)), ServerBean.class);
// for (ServerBean sb : list) {
// if (sb.getDbType().equals("mongo")) {
// if (sb.getDbUsername() != null && !sb.getDbUsername().equals("")) {
// DBCollection con = new MongoConnect(sb.getDbHost(), sb.getDbPort(), sb.getDbUsername(),
// sb.getDbPassword(), sb.getDbName()).getCollection(sb.getCollection());
// DbDepot.dbCons.put(sb.getOther(), con);
// } else {
// DBCollection con = new MongoConnect(sb.getDbHost(), sb.getDbPort(), sb.getDbName())
// .getCollection(sb.getCollection());
// DbDepot.dbCons.put(sb.getOther(), con);
// }
// }
// }
// DbDepot.nowPt = pt;
// List<ServerBean> list = mongo.find(Query.query(Criteria.where("pt").is(pt)),
// ServerBean.class);
// for (ServerBean sb : list) {
// if (sb.getDbType().equals("mongo")) {
// if (sb.getDbUsername() != null && !sb.getDbUsername().equals("")) {
// DBCollection con = new MongoConnect(sb.getDbHost(), sb.getDbPort(),
// sb.getDbUsername(),
// sb.getDbPassword(), sb.getDbName()).getCollection(sb.getCollection());
// DbDepot.dbCons.put(sb.getOther(), con);
// } else {
// DBCollection con = new MongoConnect(sb.getDbHost(), sb.getDbPort(),
// sb.getDbName())
// .getCollection(sb.getCollection());
// DbDepot.dbCons.put(sb.getOther(), con);
// }
// }
// }
// DbDepot.nowPt = pt;
}
@Override
public List<Channel> findAllChannels(int pageNo,int pageSize) {
Query query=new Query();
query.skip((pageNo-1)*pageSize).limit(pageSize);
public List<Channel> findAllChannels(int pageNo, int pageSize) {
Query query = new Query();
query.skip((pageNo - 1) * pageSize).limit(pageSize);
List<Channel> list = mongo.find(query, Channel.class);
return list;
}
@Override
public List<Channel> findChannels() {
List<Channel> list = mongo.find(null,Channel.class);
List<Channel> list = mongo.find(null, Channel.class);
return list;
}
......@@ -133,7 +138,7 @@ public class ServerDaoImpl implements ServerDao {
@Override
public boolean deleteChannel(List<String> id) {
WriteResult result = mongo.remove(Query.query(Criteria.where("_id").in(id)), Channel.class);
return result.getN() >0 ? true : false;
return result.getN() > 0 ? true : false;
}
@Override
......@@ -161,4 +166,10 @@ public class ServerDaoImpl implements ServerDao {
return mongo.getCollection("Channel").distinct("pt");
}
@Override
public ServerBean findServerBean(String pt) {
return mongo.findOne(Query.query(Criteria.where("pt").is(pt).andOperator(Criteria.where("dbType").is("Elasticsearch"))),
ServerBean.class);
}
}
src/main/java/com/zhiwei/manage/handle/DbDepot.java
......@@ -8,22 +8,22 @@ import org.apache.commons.logging.LogFactory;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.stereotype.Service;
import com.zhiwei.manage.bean.ServerBean;
@Service("firstRun")
public class DbDepot {
@Resource(name = "mongoTemplate")
private MongoTemplate mongo;
public static ConcurrentMap<String, Object> dbCons;
public static ConcurrentMap<String, Object> newsCol;
public static ConcurrentMap<String, Object> dataCol;
public static ConcurrentMap<String, Object> esClients;
public static ConcurrentMap<String, ServerBean> serverBeans;
public static int month = 0;
private static final Log log = LogFactory.getLog(DbDepot.class);
public void init() {
log.info("数据连接map初始化");
dataCol = new ConcurrentHashMap<String, Object>();
dbCons = new ConcurrentHashMap<String, Object>();
newsCol = new ConcurrentHashMap<String, Object>();
esClients = new ConcurrentHashMap<String, Object>();
serverBeans = new ConcurrentHashMap<String, ServerBean>();
System.out.println("连接初始化成功");
}
}
src/main/java/com/zhiwei/manage/handle/MainThread.java
......@@ -22,6 +22,7 @@ import com.zhiwei.manage.bean.Message;
import com.zhiwei.manage.bean.Messages;
import com.zhiwei.manage.bean.NewsDelayed;
import com.zhiwei.manage.bean.PingUrl;
import com.zhiwei.manage.bean.ServerBean;
import com.zhiwei.manage.bean.Template;
import com.zhiwei.manage.orm.BeanFactory;
import com.zhiwei.manage.service.CrawTemplateServiceImpl;
......@@ -162,8 +163,10 @@ public class MainThread extends Thread {
CrawTemplateServiceImpl crawService = BeanFactory.getBean(CrawTemplateServiceImpl.class);
DataServiceImpl dataService = BeanFactory.getBean(DataServiceImpl.class);
ServerServiceImpl serverService = BeanFactory.getBean(ServerServiceImpl.class);
MessageServiceImpl messageService=BeanFactory.getBean(MessageServiceImpl.class);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH");
List<Channel> channel = serverService.findDisChannel();
Calendar loop=Calendar.getInstance();
while (true) {
Calendar start = Calendar.getInstance();
if (start.get(Calendar.HOUR_OF_DAY) < 1) {
......@@ -179,7 +182,8 @@ public class MainThread extends Thread {
for (Channel chan : channel) {
map.put(chan.getPt(), chan.getValue());
try {
count = es.getCountByPt(start.getTime(), end.getTime(), chan.getPt(), chan.getValue());
ServerBean ser=serverService.findServer(chan.getPt());
count = es.getCountByPt(start.getTime(), end.getTime(), chan.getPt(), chan.getValue(),ser);
System.out.println(count);
Data data = new Data();
data.setCount(count);
......@@ -206,7 +210,7 @@ public class MainThread extends Thread {
int cwCount = 0;
String pt = crawService.findPt(tmp.getSpyderInfoId());
cwCount = (int) es.getCounts(start.getTime(), end.getTime(), pt, map.get(pt),
tmp.getSpyderInfoId());
tmp.getSpyderInfoId(),serverService.findServer(pt));
try {
Data data = new Data();
data.setCount(cwCount);
......@@ -214,6 +218,20 @@ public class MainThread extends Thread {
data.setTempName(tmp.getTempName());
dataService.insert(data);
log.info(TimeUtil.yearToDay(start.getTime()) + "|" + tmp.getTempName() + "|入库");
if(cwCount==0) {
loop.setTime(start.getTime());
loop.add(Calendar.DATE, -2);
cwCount = (int) es.getCounts(start.getTime(), end.getTime(), pt, map.get(pt),
tmp.getSpyderInfoId(),serverService.findServer(pt));
if(cwCount<3) {
Message msg=new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg("连续三天数据不足3条，请确认采集是否有问题");
msg.setPt(pt);
msg.setTemplateLv(1);
messageService.insert(msg);
}
}
} catch (Exception e) {
log.error(tmp.getTempName() + "入库出错，错误信息{}", e);
}
......
src/main/java/com/zhiwei/manage/service/DataService.java
......@@ -15,30 +15,21 @@ public interface DataService {
public List<Data> findByPtDay(Date startTime, Date endTime, String pt);
public List<Data> findBySource(Date startTime, Date endTime, String source);
public List<Data> findBySource(Date startTime, Date endTime,String source);
public List<Data> findBytempName(Date startTime, Date endTime, String tempName);
public List<Data> findByMethod(Date startTime, Date endTime, String method, Object value);
// public List<Data> findByMethod(Date startTime, Date endTime, String method, Object value);
public int findBySourceFromLibrary(Date startTime, Date endTime, String source);
// public int findBySourceFromLibrary(Date startTime, Date endTime, String source);
public List<EasyNews> findByAccurateSource(String startTime, String endTime, String source);
public List<EasyNews> findByAccurateSpId(String startTime, String endTime, String SpId);
public int findByAccurateType(Date startTime, Date endTime, String type);
public int findByTempName(Date startTime, Date endTime, String tempName);
public List<String> findSourceByLikeParam(String param);
public List<Data> findByMethodAndParam(Date startTime, Date endTime,String method,String param);
public List<String> findTmpNameByLikeParam(String param);
public List<String> findTypeByLikeParam(String param);
// 根据配置获取数据
public long finCountByDayData(Date startDate, Date endDate, Template temp);
public long finCountByDayData(Date startDate, Date endDate, String pt);
// public long finCountByDayData(Date startDate, Date endDate, Template temp);
//
// public long finCountByDayData(Date startDate, Date endDate, String pt);
}
src/main/java/com/zhiwei/manage/service/DataServiceImpl.java
......@@ -9,15 +9,21 @@ import org.springframework.stereotype.Service;
import com.zhiwei.manage.bean.Data;
import com.zhiwei.manage.bean.EasyNews;
import com.zhiwei.manage.bean.Template;
import com.zhiwei.manage.dao.CrawTemplateDao;
import com.zhiwei.manage.dao.DataDao;
import com.zhiwei.manage.dao.ServerDao;
import com.zhiwei.manage.util.ESUtil;
@Service
public class DataServiceImpl implements DataService {
@Autowired
private DataDao dataDao;
@Autowired
private ServerDao serverDao;
@Autowired
private CrawTemplateService crService;
private ESUtil es=new ESUtil();
@Override
public void insert(Data data) {
dataDao.insert(data);
......@@ -39,63 +45,18 @@ public class DataServiceImpl implements DataService {
}
@Override
public List<EasyNews> findByAccurateSource(String startTime, String endTime, String source) {
return dataDao.findByAccurateSource(startTime, endTime, source);
}
@Override
public List<EasyNews> findByAccurateSpId(String startTime, String endTime, String SpId) {
return dataDao.findByAccurateSpId(startTime, endTime, SpId);
}
@Override
public int findBySourceFromLibrary(Date startTime, Date endTime, String source) {
return dataDao.findBySourceFromLibrary(startTime, endTime, source);
}
@Override
public List<String> findSourceByLikeParam(String param) {
return dataDao.findSourceByLikeParam(param);
}
@Override
public List<String> findTmpNameByLikeParam(String param) {
return dataDao.findTmpNameByLikeParam(param);
}
@Override
public int findByAccurateType(Date startTime, Date endTime, String type) {
return dataDao.findByAccurateType(startTime, endTime, type);
}
@Override
public List<String> findTypeByLikeParam(String param) {
return dataDao.findTypeByLikeParam(param);
}
@Override
public List<Data> findByMethod(Date startTime, Date endTime, String method, Object value) {
return dataDao.findByMethod(startTime, endTime, method, value);
}
@Override
public long finCountByDayData(Date startTime, Date endTime, Template temp) {
return dataDao.finCountByDayData(startTime, endTime, temp);
}
@Override
public int findByTempName(Date startTime, Date endTime, String tempName) {
return dataDao.findBySpId(startTime, endTime, tempName);
}
@Override
public long finCountByDayData(Date startDate, Date endDate, String pt) {
return dataDao.finCountByDayData(startDate, endDate, pt);
public List<Data> findByPtDay(Date startTime, Date endTime, String pt) {
return dataDao.findByPtDay(startTime, endTime, pt);
}
@Override
public List<Data> findByPtDay(Date startTime, Date endTime, String pt) {
return dataDao.findByPtDay(startTime, endTime, pt);
public List<Data> findByMethodAndParam(Date startTime, Date endTime, String method, String param) {
return es.findByMethodAndParam(crService,startTime, endTime, method,param,serverDao.findServerBean("网媒"));
}
}
src/main/java/com/zhiwei/manage/service/ServerService.java
......@@ -31,5 +31,7 @@ public interface ServerService {
public List<Channel> findDisChannel();
public List<String> findDisChan();
public ServerBean findServer(String pt);
}
src/main/java/com/zhiwei/manage/service/ServerServiceImpl.java
......@@ -103,4 +103,9 @@ public class ServerServiceImpl implements ServerService {
return null;
}
@Override
public ServerBean findServer(String pt) {
return sd.findServerBean(pt);
}
}
src/main/java/com/zhiwei/manage/servlet/DataController.java
package com.zhiwei.manage.servlet;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
......@@ -19,20 +17,14 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.multipart.support.RequestPartServletServerHttpRequest;
import com.alibaba.fastjson.JSON;
import com.zhiwei.manage.bean.Channel;
import com.zhiwei.manage.bean.Data;
import com.zhiwei.manage.bean.EasyNews;
import com.zhiwei.manage.bean.JsonResult;
import com.zhiwei.manage.service.CrawTemplateService;
import com.zhiwei.manage.service.DataService;
import com.zhiwei.manage.service.ServerService;
import com.zhiwei.manage.util.Change;
import com.zhiwei.manage.util.TimeUtil;
import com.zhiwei.manage.util.Tools;
@Controller
......@@ -44,8 +36,6 @@ public class DataController {
private DataService dataService;
@Autowired
private ServerService serverService;
@Autowired
private CrawTemplateService crService;
@RequestMapping(value = "/getData", produces = "application/json;charset=utf-8")
@ResponseBody
......@@ -84,46 +74,20 @@ public class DataController {
if (method == null) {
JsonResult result = new JsonResult(false, 200, "参数不全", "");
return JSON.toJSONString(result);
} else if (timeType.equals("hour")) {
// 小时数据
} else if (timeType.equals("hour")) {// 具体查询数据
if (start == 0) {
startd.set(Calendar.HOUR, -10);
} else {
startd.setTimeInMillis(start);
endTime.setTimeInMillis(start);
startd.add(Calendar.HOUR_OF_DAY, -8);
endTime.setTimeInMillis(end);
endTime.add(Calendar.HOUR_OF_DAY, -8);
}
Calendar ds = Calendar.getInstance();
ds.setTimeInMillis(end);
String source = "";
String source = (String) map.get(method);
List<Data> list = new ArrayList<Data>();
startd.add(Calendar.HOUR, -8);
endTime.add(Calendar.HOUR, -7);
ds.add(Calendar.HOUR, -6);
String spid = crService.getSpIdByConfigName((String) map.get("tempName"));
while (endTime.getTime().before(ds.getTime())) {
endTime.add(Calendar.HOUR, 1);
int count = 0;
if (method.equals("type")) {
source = (String) map.get("type");
count = dataService.findByAccurateType(startd.getTime(), endTime.getTime(), source);
} else {
if (method.equals("source")) {
source = (String) map.get("source");
count = dataService.findBySourceFromLibrary(startd.getTime(), endTime.getTime(), source);
} else {
source = (String) map.get("tempName");
count = dataService.findByTempName(startd.getTime(), endTime.getTime(), spid);
}
}
Data data = new Data();
data.setCount(count);
data.setSource(source);
data.setTime(startd.getTime());
list.add(data);
startd.add(Calendar.HOUR, 1);
}
list = dataService.findByMethodAndParam(startd.getTime(), endTime.getTime(), method, source);
return JSON.toJSONString(Tools.beResult(list, source));
} else {
} else {// day,数据结果从库里拿
if (start == 0) {
start = startd.getTimeInMillis() - 1000 * 3600 * 24 * 7;
startd.setTimeInMillis(start);
......@@ -137,10 +101,12 @@ public class DataController {
String source = (String) map.get("source");
list = dataService.findBySource(startd.getTime(), endTime.getTime(), source);
channel.setValue(source);
} else {
} else if (method.equals("tempName")) {
String tempName = (String) map.get("tempName");
list = dataService.findBytempName(startd.getTime(), endTime.getTime(), tempName);
channel.setValue(tempName);
} else {
}
channel.setList(list);
JsonResult result = new JsonResult(true, 200, "返回成功", channel);
......@@ -187,6 +153,7 @@ public class DataController {
return JSON.toJSONString(result);
}
} catch (Exception e) {
log.error("处理错误，错误信息{}",e);
JsonResult result = new JsonResult(false, 500, e.getMessage(), "");
return JSON.toJSONString(result);
}
......@@ -197,9 +164,9 @@ public class DataController {
public String getDayDataByChannel(@RequestBody Map<String, Object> map) {
long start = (long) map.get("start");
long end = (long) map.get("end");
String pt = String.valueOf(map.get("pt")) ;
String pt = String.valueOf(map.get("pt"));
try {
Calendar nows=Calendar.getInstance();
Calendar nows = Calendar.getInstance();
Calendar startTime = Calendar.getInstance();
startTime.setTimeInMillis(start);
Calendar endTime = Calendar.getInstance();
......@@ -212,6 +179,7 @@ public class DataController {
JsonResult result = new JsonResult(true, 200, "请求成功", datas);
return JSON.toJSONString(result);
} catch (Exception e) {
log.error("处理错误，错误信息{}",e);
JsonResult result = new JsonResult(false, 500, e.getMessage(), "");
return JSON.toJSONString(result);
}
......@@ -219,7 +187,7 @@ public class DataController {
@RequestMapping(value = "/getSource", produces = "application/json;charset=utf-8")
@ResponseBody
public String getAllDataCountByTime(@RequestBody Map<String, Object> map) {
public String getSource(@RequestBody Map<String, Object> map, HttpServletRequest req) {
String value = (String) map.get("value");
String method = (String) map.get("method");
if (method != null) {
......@@ -227,10 +195,6 @@ public class DataController {
List<String> list = dataService.findTmpNameByLikeParam(value);
JsonResult result = new JsonResult(true, 200, "请求成功", list);
return JSON.toJSONString(result);
} else if (method.equals("type")) {
List<String> list = dataService.findTypeByLikeParam(value);
JsonResult result = new JsonResult(true, 200, "请求成功", list);
return JSON.toJSONString(result);
} else {
JsonResult result = new JsonResult(true, 200, "请求成功", new ArrayList<String>());
return JSON.toJSONString(result);
......
src/main/java/com/zhiwei/manage/util/ESClient.java
......@@ -15,33 +15,21 @@ import org.elasticsearch.transport.client.PreBuiltTransportClient;
*
*/
public class ESClient {
private static String esIp;
private static int esPort;
private static String clusterName;
static TransportClient client=null;
private static class ESClientHolder {
static TransportClient client = initESClient();
private static TransportClient initESClient() {
esIp=Config.getVal("ESIP");
clusterName=Config.getVal("clusterName");
esPort=Integer.valueOf(Config.getVal("ESPort"));
Settings esSettings = Settings.builder().put("cluster.name", clusterName) // 设置ES实例的名称
// 自动嗅探整个集群的状态，把集群中其他ES节点的ip添加到本地的客户端列表中
.put("client.transport.sniff", false).build();
TransportClient client = new PreBuiltTransportClient(esSettings);
try {
client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(esIp), esPort));
} catch (UnknownHostException e) {
e.printStackTrace();
}
return client;
public ESClient(String clusterName,String esIp,int esPort) {
Settings esSettings = Settings.builder().put("cluster.name", clusterName) // 设置ES实例的名称
.put("client.transport.sniff", false).build();
client = new PreBuiltTransportClient(esSettings);
try {
client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(esIp), esPort));
} catch (UnknownHostException e) {
e.printStackTrace();
}
}
public static TransportClient getInstance() {
return ESClientHolder.client;
public TransportClient getClient() {
return client;
}
}
src/main/java/com/zhiwei/manage/util/ESUtil.java
......@@ -20,10 +20,16 @@ import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.zhiwei.manage.bean.Data;
import com.zhiwei.manage.bean.EasyNews;
import com.zhiwei.manage.bean.ServerBean;
import com.zhiwei.manage.handle.DbDepot;
import com.zhiwei.manage.handle.MainThread;
import com.zhiwei.manage.orm.BeanFactory;
import com.zhiwei.manage.service.CrawTemplateService;
/**
* es工具类
......@@ -33,7 +39,7 @@ import com.zhiwei.manage.handle.MainThread;
@Component
public class ESUtil {
private static final Logger log = LoggerFactory.getLogger(ESUtil.class);
static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
final static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
/**
* 获取匹配时终止时间
......@@ -60,7 +66,8 @@ public class ESUtil {
* @param spyderInfoId
* @return
*/
public int getDatas(Date start, Date end, String pt, String type, String value, String spyderInfoId) {
public int getDatas(Date start, Date end, String pt, String type, String value, String spyderInfoId,
ServerBean ser) {
Calendar startTime = Calendar.getInstance();
startTime.setTime(start);
startTime.add(Calendar.HOUR, -8);
......@@ -68,7 +75,7 @@ public class ESUtil {
endTime.setTime(end);
endTime.add(Calendar.HOUR, -8);
try {
Client esClient = ESClient.getInstance();
Client esClient = new ESClient(ser.getCollection(), ser.getDbHost(), ser.getDbPort()).getClient();
// 搜索数据
SearchRequestBuilder srb = esClient.prepareSearch(Config.getVal(MainThread.map.get(pt) + ".indexName"));
BoolQueryBuilder bb = new BoolQueryBuilder();
......@@ -79,6 +86,7 @@ public class ESUtil {
srb.setSize(1);
SearchResponse response = srb.execute().actionGet();
SearchHits searchHits = response.getHits();
esClient.close();
return (int) searchHits.getTotalHits();
} catch (Exception e) {
log.error("es检索出错，错误信息【{}】", e);
......@@ -87,31 +95,116 @@ public class ESUtil {
}
public static void main(String[] args) throws ParseException {
System.out.println(new ESUtil().getCountByPt(sdf.parse("2018-04-19"), new Date(), "网媒", "media"));
Client esClient = null;
Calendar cal=Calendar.getInstance();
cal.add(Calendar.HOUR_OF_DAY, -24);
esClient = new ESClient("Media-University","1.119.44.201", 9300).getClient();
SearchRequestBuilder srb = esClient.prepareSearch("mediaspider*");
BoolQueryBuilder bb = new BoolQueryBuilder();
srb.setQuery(bb);
QueryBuilder matchQuery = QueryBuilders.matchPhraseQuery("type", "今日头条");
bb.should(matchQuery);
srb.setSize(1);
srb.setPostFilter(QueryBuilders.rangeQuery("time").gte(getNeedTime(cal)).lte(getNeedTime(Calendar.getInstance())));
SearchResponse response = srb.execute().actionGet();
SearchHits searchHits = response.getHits();
System.out.println(searchHits.getTotalHits());
}
public static String getNeedTime(Calendar cal) {
Calendar time=Calendar.getInstance();
time.setTime(cal.getTime());
time.add(Calendar.HOUR_OF_DAY, -8);
String result="";
result=sdf.format(time.getTime())+"T"
+(time.get(Calendar.HOUR_OF_DAY)>10?time.get(Calendar.HOUR_OF_DAY):"0"+time.get(Calendar.HOUR_OF_DAY)
+":00:00.000Z");
return result;
}
/**
* 根据时间和参数查询具体数据量
* @param start
* @param end
* @param method
* @param param
* @param ser
* @return
*/
public List<Data> findByMethodAndParam(CrawTemplateService crService,Date start, Date end, String method, String param, ServerBean ser) {
List<Data> list=new ArrayList<>();
Calendar startTime = Calendar.getInstance();
startTime.setTime(start);
Calendar endTime = Calendar.getInstance();
endTime.setTime(end);
String key="";
String value="";
if(method.equals("tempName")) {
key="spyderInfoId";
value = crService.getSpIdByConfigName(param);
}else {
key=method;
value=param;
}
Client esClient = null;
try {
if (DbDepot.esClients.get(ser.getPt()) == null) {
esClient = new ESClient(ser.getDbName(), ser.getDbHost(), ser.getDbPort()).getClient();
DbDepot.esClients.put(ser.getPt(), esClient);
} else {
esClient = (Client) DbDepot.esClients.get(ser.getPt());
}
Calendar loopTime=Calendar.getInstance();
loopTime.setTime(startTime.getTime());
loopTime.add(Calendar.HOUR_OF_DAY, 1);
endTime.add(Calendar.HOUR_OF_DAY, 1);//n+1
while (startTime.getTime().before(endTime.getTime())) {
// 搜索数据
SearchRequestBuilder srb = esClient.prepareSearch(ser.getCollection());
BoolQueryBuilder bb = new BoolQueryBuilder();
QueryBuilder matchQuery = QueryBuilders.matchPhraseQuery(key, value);
bb.should(matchQuery);
srb.setQuery(bb);
srb.setSize(1);
srb.setPostFilter(QueryBuilders.rangeQuery("time").gte(getNeedTime(startTime)).lte(getNeedTime(loopTime)));
SearchResponse response = srb.execute().actionGet();
SearchHits searchHits = response.getHits();
Data data=new Data();
data.setSource(method);
data.setCount(searchHits.getTotalHits());
data.setTime(startTime.getTime());
list.add(data);
startTime.add(Calendar.HOUR_OF_DAY, 1);
loopTime.add(Calendar.HOUR_OF_DAY, 1);
}
} catch (Exception e) {
log.error("es检索出错，错误信息【{}】", e);
}
return list;
}
public int getCounts(Date start, Date end, String pt, String value, String spid) {
public int getCounts(Date start, Date end, String pt, String value, String spid, ServerBean ser) {
Calendar startTime = Calendar.getInstance();
startTime.setTime(start);
startTime.add(Calendar.HOUR, -8);
// startTime.add(Calendar.HOUR, -8);
Calendar endTime = Calendar.getInstance();
endTime.setTime(end);
endTime.add(Calendar.HOUR, -8);
// endTime.add(Calendar.HOUR, -8);
try {
Client esClient = ESClient.getInstance();
Client esClient = new ESClient(ser.getDbName(), ser.getDbHost(), ser.getDbPort()).getClient();
// 搜索数据
System.out.println(pt);
System.out.println(Config.getVal(MainThread.map.get(pt) + ".indexName"));
SearchRequestBuilder srb = esClient.prepareSearch(Config.getVal(MainThread.map.get(pt) + ".indexName"));
System.out.println(ser.getCollection());
SearchRequestBuilder srb = esClient.prepareSearch(ser.getCollection());
BoolQueryBuilder bb = new BoolQueryBuilder();
srb.setQuery(bb);
addQuerys(pt, startTime, endTime, srb, bb);
srb.setPostFilter(QueryBuilders.rangeQuery("time").gte(getNeedTime(startTime)).lte(getNeedTime(endTime)));
QueryBuilder matchQuery = QueryBuilders.matchPhraseQuery("spyderInfoId", spid);
bb.should(matchQuery);
srb.setSize(1);
SearchResponse response = srb.execute().actionGet();
SearchHits searchHits = response.getHits();
esClient.close();
return (int) searchHits.getTotalHits();
} catch (Exception e) {
log.error("es检索出错，错误信息【{}】", e);
......@@ -128,7 +221,7 @@ public class ESUtil {
* @param value
* @return
*/
public int getCountByPt(Date start, Date end, String pt, String value) {
public int getCountByPt(Date start, Date end, String pt, String value, ServerBean ser) {
Calendar startTime = Calendar.getInstance();
startTime.setTime(start);
startTime.add(Calendar.HOUR, -8);
......@@ -136,9 +229,9 @@ public class ESUtil {
endTime.setTime(end);
endTime.add(Calendar.HOUR, -8);
try {
Client esClient = ESClient.getInstance();
Client esClient = new ESClient(ser.getDbName(), ser.getDbHost(), ser.getDbPort()).getClient();
// 搜索数据
SearchRequestBuilder srb = esClient.prepareSearch(Config.getVal(value + ".indexName"));
SearchRequestBuilder srb = esClient.prepareSearch(ser.getCollection());
BoolQueryBuilder bb = new BoolQueryBuilder();
srb.setQuery(bb);
addQuerys(pt, startTime, endTime, srb, bb);
......@@ -152,7 +245,7 @@ public class ESUtil {
return 0;
}
private void addQuerys(String pt, Calendar startTime, Calendar endTime, SearchRequestBuilder srb,
private static void addQuerys(String pt, Calendar startTime, Calendar endTime, SearchRequestBuilder srb,
BoolQueryBuilder bb) {
if (pt.equals("微博")) {
srb.setPostFilter(QueryBuilders.rangeQuery("ins").gte(sendTime(startTime)).lt(sendTime(endTime)));
......@@ -185,35 +278,4 @@ public class ESUtil {
: "0" + time.get(Calendar.HOUR_OF_DAY))
+ ":00:00.000Z";
}
public static List<String> getNewsForWechatWithMechanism(Date start, Date end) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd日HH:mm:ss");
Calendar startTime = Calendar.getInstance();
// startTime.setTime(start);
// startTime.add(Calendar.HOUR, -8);
// Calendar endTime = Calendar.getInstance();
// endTime.setTime(end);
// endTime.add(Calendar.HOUR, -8);
try {
log.info("检索开始");
Client esClient = ESClient.getInstance();
// 搜索数据
SearchRequestBuilder srb = esClient.prepareSearch(Config.getVal("weibo.indexName"));
BoolQueryBuilder bb = new BoolQueryBuilder();
// QueryBuilder matchQuery2 = QueryBuilders.matchPhraseQuery("pt", "");
// bb.should(matchQuery2);
srb.setQuery(bb);
srb.setPostFilter(
QueryBuilders.rangeQuery("ins").gte("2018-04-17T00:00:00.000Z").lt("2018-04-18T00:00:00.000Z"));
srb.setSize(1);
SearchResponse response = srb.execute().actionGet();
SearchHits searchHits = response.getHits();
System.out.println(searchHits.getTotalHits());
log.info("结束");
esClient.close();
} catch (Exception e) {
log.error("es检索出错，错误信息【{}】", e);
}
return null;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment