Commit f0e1ed56 by shenjinzhu

模板检测修改

parent 6d9ea76b
...@@ -6,6 +6,7 @@ import java.util.List; ...@@ -6,6 +6,7 @@ import java.util.List;
import com.zhiwei.manage.bean.Data; import com.zhiwei.manage.bean.Data;
import com.zhiwei.manage.bean.EasyNews; import com.zhiwei.manage.bean.EasyNews;
import com.zhiwei.manage.bean.News; import com.zhiwei.manage.bean.News;
import com.zhiwei.manage.bean.Template;
public interface DataDao { public interface DataDao {
...@@ -19,8 +20,8 @@ public interface DataDao { ...@@ -19,8 +20,8 @@ public interface DataDao {
public List<Data> findByMethod(Date startTime,Date endTime,String method,Object value); public List<Data> findByMethod(Date startTime,Date endTime,String method,Object value);
public int findBySourceFromLibrary(Date startTime, Date endTime, String source); public int findBySourceFromLibrary(Date startTime, Date endTime, String source);
public int findBySpId(Date startTime, Date endTime, String syId); public int findBySpId(Date startTime, Date endTime, String syId);
/** /**
* 根据数据来源来搜索数据 * 根据数据来源来搜索数据
...@@ -31,9 +32,8 @@ public interface DataDao { ...@@ -31,9 +32,8 @@ public interface DataDao {
*/ */
public List<EasyNews> findByAccurateSource(String startTime, String endTime, String source); public List<EasyNews> findByAccurateSource(String startTime, String endTime, String source);
public int findByAccurateType(Date startTime, Date endTime, String type); public int findByAccurateType(Date startTime, Date endTime, String type);
/** /**
* 根据可视化模板的id去搜索数据 * 根据可视化模板的id去搜索数据
* @param startTime * @param startTime
...@@ -43,25 +43,13 @@ public interface DataDao { ...@@ -43,25 +43,13 @@ public interface DataDao {
*/ */
public List<EasyNews> findByAccurateSpId(String startTime, String endTime, String SpId); public List<EasyNews> findByAccurateSpId(String startTime, String endTime, String SpId);
/**
* 微博数据统计
* @param startTime
* @param endTime
* @return
*/
public long findWeiboCount(Date startTime,Date endTime);
public long findWeixinCount(Date startTime,Date endTime,String pt);
public List<String> findSourceByLikeParam(String param); public List<String> findSourceByLikeParam(String param);
public List<String> findTmpNameByLikeParam(String param); public List<String> findTmpNameByLikeParam(String param);
public List<String> findTypeByLikeParam(String param); public List<String> findTypeByLikeParam(String param);
public long finCountByDayData(Date startTime,Date endTime,String pt,String id); public long finCountByDayData(Date startTime,Date endTime,Template temp);
public long finCountByDayData(Date startTime,Date endTime,String pt);
} }
...@@ -33,10 +33,12 @@ import com.zhiwei.manage.bean.Data; ...@@ -33,10 +33,12 @@ import com.zhiwei.manage.bean.Data;
import com.zhiwei.manage.bean.EasyNews; import com.zhiwei.manage.bean.EasyNews;
import com.zhiwei.manage.bean.News; import com.zhiwei.manage.bean.News;
import com.zhiwei.manage.bean.ServerBean; import com.zhiwei.manage.bean.ServerBean;
import com.zhiwei.manage.bean.Template;
import com.zhiwei.manage.bean.Weibo; import com.zhiwei.manage.bean.Weibo;
import com.zhiwei.manage.handle.DbDepot; import com.zhiwei.manage.handle.DbDepot;
import com.zhiwei.manage.util.Change; import com.zhiwei.manage.util.Change;
import com.zhiwei.manage.util.MongoConnect; import com.zhiwei.manage.util.MongoConnect;
@SuppressWarnings({ "all" }) @SuppressWarnings({ "all" })
@Component @Component
public class DataDaoImpl implements DataDao { public class DataDaoImpl implements DataDao {
...@@ -86,7 +88,8 @@ public class DataDaoImpl implements DataDao { ...@@ -86,7 +88,8 @@ public class DataDaoImpl implements DataDao {
Criteria criteria = new Criteria(); Criteria criteria = new Criteria();
criteria.and("source").regex(source); criteria.and("source").regex(source);
criteria.and("time").gte(startTime).lt(endTime); criteria.and("time").gte(startTime).lt(endTime);
return mongo2.find(Query.query(criteria), EasyNews.class, "net_media_" + year + (month>9?""+month:"0"+month)); return mongo2.find(Query.query(criteria), EasyNews.class,
"net_media_" + year + (month > 9 ? "" + month : "0" + month));
} }
@Override @Override
...@@ -97,22 +100,12 @@ public class DataDaoImpl implements DataDao { ...@@ -97,22 +100,12 @@ public class DataDaoImpl implements DataDao {
Criteria criteria = new Criteria(); Criteria criteria = new Criteria();
criteria.and("spyderInfoId").is(SpId); criteria.and("spyderInfoId").is(SpId);
criteria.and("time").gt(startTime).lt(endTime); criteria.and("time").gt(startTime).lt(endTime);
return mongo2.find(Query.query(criteria), EasyNews.class, "net_media_" + year + (month>9?""+month:"0"+month)); return mongo2.find(Query.query(criteria), EasyNews.class,
} "net_media_" + year + (month > 9 ? "" + month : "0" + month));
@Override
public long findWeiboCount(Date startTime, Date endTime) {
Calendar c = Calendar.getInstance();
c.setTime(startTime);
int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1;
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");
return weibo.count(Query.query(Criteria.where("time").gte(sdf.format(startTime)).lt(sdf.format(endTime))),
Weibo.class, "status_" + year + (month>9?""+month:"0"+month));
} }
@Override @Override
public long finCountByDayData(Date startTime, Date endTime, String pt, String id) { public long finCountByDayData(Date startTime, Date endTime, String pt) {
String end = ""; String end = "";
Calendar cal = Calendar.getInstance(); Calendar cal = Calendar.getInstance();
cal.setTime(startTime); cal.setTime(startTime);
...@@ -123,7 +116,7 @@ public class DataDaoImpl implements DataDao { ...@@ -123,7 +116,7 @@ public class DataDaoImpl implements DataDao {
col = (MongoCollection<Document>) DbDepot.dataCol.get(pt); col = (MongoCollection<Document>) DbDepot.dataCol.get(pt);
} }
} }
int resultCount=0; int resultCount = 0;
if (col == null) { if (col == null) {
ServerBean ser = mongo.findOne( ServerBean ser = mongo.findOne(
Query.query(Criteria.where("pt").is(pt).andOperator(Criteria.where("other").is("data"))), Query.query(Criteria.where("pt").is(pt).andOperator(Criteria.where("other").is("data"))),
...@@ -133,30 +126,65 @@ public class DataDaoImpl implements DataDao { ...@@ -133,30 +126,65 @@ public class DataDaoImpl implements DataDao {
MongoDatabase mongo = mongoClient.getDatabase(ser.getDbName()); MongoDatabase mongo = mongoClient.getDatabase(ser.getDbName());
for (int i = 1; i < 6; i++) { for (int i = 1; i < 6; i++) {
col = mongo.getCollection(ser.getCollection() + i); col = mongo.getCollection(ser.getCollection() + i);
Bson query = new BasicDBObject("update_at", new BasicDBObject("$gte", startTime).append("$lte", endTime)); Bson query = new BasicDBObject("update_at",
resultCount+=col.count(query); new BasicDBObject("$gte", startTime).append("$lte", endTime));
resultCount += col.count(query);
} }
} }
return resultCount; return resultCount;
} }
col = getcol(startTime, endTime, pt);
long count = 0;
if (pt.equals("微博")) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");
Bson query = new BasicDBObject("time",
new BasicDBObject("$gte", sdf.format(startTime)).append("$lte", sdf.format(endTime)));
count = col.count(query);
} else if (pt.equals("微信") || pt.equals("网媒") || pt.equals("论坛") || pt.equals("贴吧") || pt.equals("平媒")) {
Bson query = new BasicDBObject("time", new BasicDBObject("$gte", startTime).append("$lte", endTime))
.append("pt", pt);
count = col.count(query);
} else {
Bson query = new BasicDBObject("time", new BasicDBObject("$gte", startTime).append("$lte", endTime));
count = col.count(query);
}
return count;
}
@Override
public long finCountByDayData(Date startTime, Date endTime, Template temp) {
String end = "";
Calendar cal = Calendar.getInstance();
String pt = temp.getPt();
cal.setTime(startTime);
MongoCollection<Document> col = getcol(startTime, endTime, pt);
Bson query = new BasicDBObject("time", new BasicDBObject("$gte", startTime).append("$lte", endTime))
.append("spyderInfoId", temp.getSpyderInfoId());
return col.count(query);
}
public MongoCollection<Document> getcol(Date startTime, Date endTime, String pt) {
String end = "";
Calendar cal = Calendar.getInstance();
cal.setTime(startTime);
if (DbDepot.dataCol != null) { if (DbDepot.dataCol != null) {
if (DbDepot.dataCol.get(pt) != null && DbDepot.month == cal.get(Calendar.MONTH)) { if (DbDepot.dataCol.get(pt) != null && DbDepot.month == cal.get(Calendar.MONTH)) {
col = (MongoCollection<Document>) DbDepot.dataCol.get(pt); return (MongoCollection<Document>) DbDepot.dataCol.get(pt);
} else { } else {
ServerBean ser = mongo.findOne( ServerBean ser = mongo.findOne(
Query.query(Criteria.where("pt").is(pt).andOperator(Criteria.where("other").is("data"))), Query.query(Criteria.where("pt").is(pt).andOperator(Criteria.where("other").is("data"))),
ServerBean.class); ServerBean.class);
if (ser.isTable()) { if (ser.isTable()) {
int month=cal.get(Calendar.MONTH)+1; int month = cal.get(Calendar.MONTH) + 1;
end = cal.get(Calendar.YEAR) +""+ (month>9?""+month:"0"+month); end = cal.get(Calendar.YEAR) + "" + (month > 9 ? "" + month : "0" + month);
System.out.println(end); System.out.println(end);
} }
MongoClient mongoClient = new MongoClient(ser.getDbHost(), ser.getDbPort()); MongoClient mongoClient = new MongoClient(ser.getDbHost(), ser.getDbPort());
// 连接到数据库 // 连接到数据库
MongoDatabase mongo = mongoClient.getDatabase(ser.getDbName()); MongoDatabase mongo = mongoClient.getDatabase(ser.getDbName());
col = mongo.getCollection(ser.getCollection() + end); DbDepot.dataCol.put(pt, mongo.getCollection(ser.getCollection() + end));
DbDepot.dataCol.put(pt, col);
DbDepot.month = cal.get(Calendar.MONTH); DbDepot.month = cal.get(Calendar.MONTH);
return mongo.getCollection(ser.getCollection() + end);
} }
} else { } else {
DbDepot.dataCol = new ConcurrentHashMap<String, Object>(); DbDepot.dataCol = new ConcurrentHashMap<String, Object>();
...@@ -164,36 +192,14 @@ public class DataDaoImpl implements DataDao { ...@@ -164,36 +192,14 @@ public class DataDaoImpl implements DataDao {
Query.query(Criteria.where("pt").is(pt).andOperator(Criteria.where("other").is("data"))), Query.query(Criteria.where("pt").is(pt).andOperator(Criteria.where("other").is("data"))),
ServerBean.class); ServerBean.class);
if (ser.isTable()) { if (ser.isTable()) {
int month=cal.get(Calendar.MONTH)+1; int month = cal.get(Calendar.MONTH) + 1;
end = cal.get(Calendar.YEAR) +""+ (month>9?""+month:"0"+month); end = cal.get(Calendar.YEAR) + "" + (month > 9 ? "" + month : "0" + month);
} }
MongoClient mongoClient = new MongoClient(ser.getDbHost(), ser.getDbPort()); MongoClient mongoClient = new MongoClient(ser.getDbHost(), ser.getDbPort());
// 连接到数据库
MongoDatabase mongo = mongoClient.getDatabase(ser.getDbName()); MongoDatabase mongo = mongoClient.getDatabase(ser.getDbName());
col = mongo.getCollection(ser.getCollection() + end); DbDepot.dataCol.put(pt, mongo.getCollection(ser.getCollection() + end));
DbDepot.dataCol.put(pt, col); return mongo.getCollection(ser.getCollection() + end);
}
long count = 0;
if (id == null) {
if (pt.equals("微博")) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");
Bson query = new BasicDBObject("time",
new BasicDBObject("$gte", sdf.format(startTime)).append("$lte", sdf.format(endTime)));
count = col.count(query);
} else if (pt.equals("微信") || pt.equals("网媒") || pt.equals("论坛") || pt.equals("贴吧")||pt.equals("平媒")) {
Bson query = new BasicDBObject("time", new BasicDBObject("$gte", startTime).append("$lte", endTime))
.append("pt", pt);
count = col.count(query);
} else {
Bson query = new BasicDBObject("time", new BasicDBObject("$gte", startTime).append("$lte", endTime));
count = col.count(query);
}
} else {
Bson query = new BasicDBObject("time", new BasicDBObject("$gte", startTime).append("$lte", endTime))
.append("spyderInfoId", id);
count = col.count(query);
} }
return count;
} }
public static void main(String[] args) { public static void main(String[] args) {
...@@ -209,34 +215,6 @@ public class DataDaoImpl implements DataDao { ...@@ -209,34 +215,6 @@ public class DataDaoImpl implements DataDao {
System.out.println(col.count(query)); System.out.println(col.count(query));
} }
@Override
public long findWeixinCount(Date startTime, Date endTime, String pt) {
MongoCollection<Document> col = null;
if (DbDepot.newsCol != null) {
if (DbDepot.newsCol.get(pt) != null) {
System.out.println("已经有,不重新创建");
col = (MongoCollection<Document>) DbDepot.newsCol.get(pt);
} else {
ServerBean ser = mongo.findOne(Query.query(Criteria.where("pt").is(pt)), ServerBean.class);
MongoClient mongoClient = new MongoClient(ser.getDbHost(), ser.getDbPort());
// 连接到数据库
MongoDatabase mongo = mongoClient.getDatabase(ser.getDbName());
col = mongo.getCollection(ser.getCollection());
DbDepot.newsCol.put(pt, col);
}
} else {
DbDepot.newsCol = new ConcurrentHashMap<String, Object>();
ServerBean ser = mongo.findOne(Query.query(Criteria.where("pt").is(pt)), ServerBean.class);
MongoClient mongoClient = new MongoClient(ser.getDbHost(), ser.getDbPort());
// 连接到数据库
MongoDatabase mongo = mongoClient.getDatabase(ser.getDbName());
col = mongo.getCollection(ser.getCollection());
DbDepot.newsCol.put(pt, col);
}
Bson query = new BasicDBObject("time", new BasicDBObject());
long count = col.count(query);
return count;
}
@Override @Override
public int findBySourceFromLibrary(Date startTime, Date endTime, String source) { public int findBySourceFromLibrary(Date startTime, Date endTime, String source) {
...@@ -245,7 +223,7 @@ public class DataDaoImpl implements DataDao { ...@@ -245,7 +223,7 @@ public class DataDaoImpl implements DataDao {
int year = c.get(Calendar.YEAR); int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1; int month = c.get(Calendar.MONTH) + 1;
return (int) mongo2.count(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("type").is(source)), return (int) mongo2.count(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("type").is(source)),
EasyNews.class, "net_media_" + year + (month>9?""+month:"0"+month)); EasyNews.class, "net_media_" + year + (month > 9 ? "" + month : "0" + month));
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
...@@ -255,10 +233,10 @@ public class DataDaoImpl implements DataDao { ...@@ -255,10 +233,10 @@ public class DataDaoImpl implements DataDao {
int year = c.get(Calendar.YEAR); int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1; int month = c.get(Calendar.MONTH) + 1;
try { try {
return mongo2.getCollection("net_media_" + year + (month>9?""+month:"0"+month)).distinct("source", return mongo2.getCollection("net_media_" + year + (month > 9 ? "" + month : "0" + month)).distinct("source",
new BasicDBObject("source", new BasicDBObject("$regex", param))); new BasicDBObject("source", new BasicDBObject("$regex", param)));
} catch (Exception e) { } catch (Exception e) {
return mongo2.getCollection("net_media_" + year + (month>9?""+month:"0"+month)).distinct("source", return mongo2.getCollection("net_media_" + year + (month > 9 ? "" + month : "0" + month)).distinct("source",
new BasicDBObject("source", new BasicDBObject("$regex", param))); new BasicDBObject("source", new BasicDBObject("$regex", param)));
} }
} }
...@@ -282,7 +260,7 @@ public class DataDaoImpl implements DataDao { ...@@ -282,7 +260,7 @@ public class DataDaoImpl implements DataDao {
int year = c.get(Calendar.YEAR); int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1; int month = c.get(Calendar.MONTH) + 1;
return (int) mongo2.count(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("type").is(type)), return (int) mongo2.count(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("type").is(type)),
EasyNews.class, "net_media_" + year + (month>9?""+month:"0"+month)); EasyNews.class, "net_media_" + year + (month > 9 ? "" + month : "0" + month));
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
...@@ -295,7 +273,8 @@ public class DataDaoImpl implements DataDao { ...@@ -295,7 +273,8 @@ public class DataDaoImpl implements DataDao {
DBObject obj = new BasicDBObject(); DBObject obj = new BasicDBObject();
obj.put("pt", new BasicDBObject("$ne", "微信")); obj.put("pt", new BasicDBObject("$ne", "微信"));
obj.put("type", new BasicDBObject("$regex", param)); obj.put("type", new BasicDBObject("$regex", param));
return mongo2.getCollection("net_media_" + year + (month>9?""+month:"0"+month)).distinct("type", obj); return mongo2.getCollection("net_media_" + year + (month > 9 ? "" + month : "0" + month)).distinct("type",
obj);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
...@@ -357,8 +336,9 @@ public class DataDaoImpl implements DataDao { ...@@ -357,8 +336,9 @@ public class DataDaoImpl implements DataDao {
c.setTime(startTime); c.setTime(startTime);
int year = c.get(Calendar.YEAR); int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1; int month = c.get(Calendar.MONTH) + 1;
return (int) mongo2.count(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("spyderInfoId").is(syId)), return (int) mongo2.count(
EasyNews.class, "net_media_" + year + (month>9?""+month:"0"+month)); Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("spyderInfoId").is(syId)),
EasyNews.class, "net_media_" + year + (month > 9 ? "" + month : "0" + month));
} }
} }
...@@ -45,7 +45,6 @@ public class DelayedThread extends Thread { ...@@ -45,7 +45,6 @@ public class DelayedThread extends Thread {
queue.offer(run); queue.offer(run);
cal.add(Calendar.HOUR, run.getDelayedInterval()); cal.add(Calendar.HOUR, run.getDelayedInterval());
run.setNextRun(cal.getTime()); run.setNextRun(cal.getTime());
System.out.println("开始");
} }
} }
Tools.sleep(1000); Tools.sleep(1000);
......
...@@ -46,7 +46,6 @@ public class MainThread extends Thread { ...@@ -46,7 +46,6 @@ public class MainThread extends Thread {
/** /**
* 加入执行map * 加入执行map
*
* @param template * @param template
*/ */
public static boolean pushMap(String templateId) { public static boolean pushMap(String templateId) {
...@@ -148,7 +147,6 @@ public class MainThread extends Thread { ...@@ -148,7 +147,6 @@ public class MainThread extends Thread {
ServerServiceImpl serverService = BeanFactory.getBean(ServerServiceImpl.class); ServerServiceImpl serverService = BeanFactory.getBean(ServerServiceImpl.class);
MessageServiceImpl msgService = BeanFactory.getBean(MessageServiceImpl.class); MessageServiceImpl msgService = BeanFactory.getBean(MessageServiceImpl.class);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
// int num = 1;
while (true) { while (true) {
Calendar countTest = Calendar.getInstance(); Calendar countTest = Calendar.getInstance();
Calendar start = Calendar.getInstance(); Calendar start = Calendar.getInstance();
...@@ -161,8 +159,6 @@ public class MainThread extends Thread { ...@@ -161,8 +159,6 @@ public class MainThread extends Thread {
} catch (ParseException e1) { } catch (ParseException e1) {
e1.printStackTrace(); e1.printStackTrace();
} }
// int num = 1;
// for (int i = 0; i < 19; i++) {
start.add(Calendar.DATE, -1); start.add(Calendar.DATE, -1);
countTest.add(Calendar.DATE, -7); countTest.add(Calendar.DATE, -7);
try { try {
...@@ -171,53 +167,15 @@ public class MainThread extends Thread { ...@@ -171,53 +167,15 @@ public class MainThread extends Thread {
List<Channel> channel = serverService.findDisChannel(); List<Channel> channel = serverService.findDisChannel();
for (Channel chan : channel) { for (Channel chan : channel) {
try { try {
if (chan.getPt().equals("网媒")) { count = (int) dataService.finCountByDayData(start.getTime(), end.getTime(), chan.getPt());
List<CrawTemplate> list = crawService.findAll(); Data data = new Data();
// 每日的数据量入库(对应模板) data.setCount(count);
for (CrawTemplate cr : list) { data.setPt(chan.getPt());
count = (int) dataService.finCountByDayData(start.getTime(), end.getTime(), data.setTime(start.getTime());
cr.getPt(), cr.getId()); dataService.insert(data);
Data data = new Data(); log.info(TimeUtil.yearToDay(start.getTime()) + "|" + chan.getPt() + "|入库");
data.setCount(count);
data.setTempName(cr.getConfigName());
data.setTime(start.getTime());
data.setSpyderInfoId(cr.getId());
dataService.insert(data);
if (count == 0) {
int testCount = (int) dataService.finCountByDayData(countTest.getTime(),
end.getTime(), cr.getPt(), cr.getId());
if (testCount == 0) {
Message m = new Message();
m.setHandle(false);
m.setCreateDate(start.getTime());
m.setTemplateLv(1);
m.setTempName(cr.getConfigName());
m.setPt(cr.getPt());
m.setErrorMsg("数据为0检测:" + cr.getConfigName() + "," + cr.getId() + "数据为0");
msgService.insert(m);
}
}
log.info(TimeUtil.yearToDay(start.getTime()) + "|" + data.getTempName() + "|入库");
}
count = (int) dataService.finCountByDayData(start.getTime(), end.getTime(),
chan.getPt(), null);
Data data = new Data();
data.setCount(count);
data.setPt(chan.getPt());
data.setTime(start.getTime());
dataService.insert(data);
} else {
count = (int) dataService.finCountByDayData(start.getTime(), end.getTime(),
chan.getPt(), null);
Data data = new Data();
data.setCount(count);
data.setPt(chan.getPt());
data.setTime(start.getTime());
dataService.insert(data);
log.info(TimeUtil.yearToDay(start.getTime()) + "|" + chan.getPt() + "|入库");
}
} catch (Exception e) { } catch (Exception e) {
log.error(chan.getPt()+"入库出错,错误信息{}",e);
} }
} }
Set<String> keys = allTmp.keySet(); Set<String> keys = allTmp.keySet();
...@@ -227,8 +185,7 @@ public class MainThread extends Thread { ...@@ -227,8 +185,7 @@ public class MainThread extends Thread {
if (tmp.getSpyderInfoId() != null) { if (tmp.getSpyderInfoId() != null) {
int cwCount = 0; int cwCount = 0;
String pt = crawService.findPt(tmp.getSpyderInfoId()); String pt = crawService.findPt(tmp.getSpyderInfoId());
cwCount = (int) dataService.finCountByDayData(start.getTime(), end.getTime(), tmp.getPt(), cwCount = (int) dataService.finCountByDayData(start.getTime(), end.getTime(), tmp);
tmp.getSpyderInfoId());
if (cwCount < tmp.getNewsNumber()) { if (cwCount < tmp.getNewsNumber()) {
Message m = new Message(); Message m = new Message();
m.setHandle(false); m.setHandle(false);
...@@ -246,9 +203,6 @@ public class MainThread extends Thread { ...@@ -246,9 +203,6 @@ public class MainThread extends Thread {
} finally { } finally {
mainLock.unlock(); mainLock.unlock();
} }
// end.add(Calendar.DATE, -1);
// num++;
// }
Tools.sleep(1000 * 60 * 60); Tools.sleep(1000 * 60 * 60);
} }
Tools.sleep(1000 * 30); Tools.sleep(1000 * 30);
......
...@@ -46,7 +46,6 @@ public class PingThread extends Thread { ...@@ -46,7 +46,6 @@ public class PingThread extends Thread {
PingUrl run = runMap.get(key); PingUrl run = runMap.get(key);
Calendar cal = Calendar.getInstance(); Calendar cal = Calendar.getInstance();
if (run.getNextRun().before(cal.getTime())) { if (run.getNextRun().before(cal.getTime())) {
System.out.println("ping添加一个");
queue.offer(run); queue.offer(run);
cal.add(Calendar.HOUR, run.getPingInterval()); cal.add(Calendar.HOUR, run.getPingInterval());
run.setNextRun(cal.getTime()); run.setNextRun(cal.getTime());
......
...@@ -50,7 +50,7 @@ public class Task implements Runnable { ...@@ -50,7 +50,7 @@ public class Task implements Runnable {
Template tp = MainThread.mainMap.get(ping.getTemplateId()); Template tp = MainThread.mainMap.get(ping.getTemplateId());
Message msg = new Message(); Message msg = new Message();
msg.setCreateDate(new Date()); msg.setCreateDate(new Date());
msg.setErrorMsg(code == 0 ? ping.getPingUrlOne() + "访问失败,错误代码" + code : "访问超时"); msg.setErrorMsg(code == 0 ? "访问超时":ping.getPingUrlOne() + "访问失败,错误代码" + code );
msg.setHandle(false); msg.setHandle(false);
msg.setTemplateCreator(tp.getCreator()); msg.setTemplateCreator(tp.getCreator());
msg.setTempName(ping.getTempName()); msg.setTempName(ping.getTempName());
......
...@@ -5,6 +5,7 @@ import java.util.List; ...@@ -5,6 +5,7 @@ import java.util.List;
import com.zhiwei.manage.bean.Data; import com.zhiwei.manage.bean.Data;
import com.zhiwei.manage.bean.EasyNews; import com.zhiwei.manage.bean.EasyNews;
import com.zhiwei.manage.bean.Template;
public interface DataService { public interface DataService {
...@@ -28,15 +29,12 @@ public interface DataService { ...@@ -28,15 +29,12 @@ public interface DataService {
public int findByTempName(Date startTime, Date endTime, String tempName); public int findByTempName(Date startTime, Date endTime, String tempName);
public long findWeiboCount(Date start, Date end);
public long findWeixinCount(Date start, Date end,String pt);
public List<String> findSourceByLikeParam(String param); public List<String> findSourceByLikeParam(String param);
public List<String> findTmpNameByLikeParam(String param); public List<String> findTmpNameByLikeParam(String param);
public List<String> findTypeByLikeParam(String param); public List<String> findTypeByLikeParam(String param);
//根据配置获取数据 //根据配置获取数据
public long finCountByDayData(Date startDate,Date endDate,String pt,String id); public long finCountByDayData(Date startDate,Date endDate,Template temp);
public long finCountByDayData(Date startDate,Date endDate,String pt);
} }
...@@ -8,16 +8,18 @@ import org.springframework.stereotype.Service; ...@@ -8,16 +8,18 @@ import org.springframework.stereotype.Service;
import com.zhiwei.manage.bean.Data; import com.zhiwei.manage.bean.Data;
import com.zhiwei.manage.bean.EasyNews; import com.zhiwei.manage.bean.EasyNews;
import com.zhiwei.manage.bean.Template;
import com.zhiwei.manage.dao.CrawTemplateDao; import com.zhiwei.manage.dao.CrawTemplateDao;
import com.zhiwei.manage.dao.DataDao; import com.zhiwei.manage.dao.DataDao;
@Service @Service
public class DataServiceImpl implements DataService{ public class DataServiceImpl implements DataService {
@Autowired @Autowired
private DataDao dataDao; private DataDao dataDao;
@Autowired @Autowired
private CrawTemplateDao crd; private CrawTemplateDao crd;
@Override @Override
public void insert(Data data) { public void insert(Data data) {
dataDao.insert(data); dataDao.insert(data);
...@@ -49,16 +51,6 @@ public class DataServiceImpl implements DataService{ ...@@ -49,16 +51,6 @@ public class DataServiceImpl implements DataService{
} }
@Override @Override
public long findWeiboCount(Date start, Date end) {
return dataDao.findWeiboCount(start, end);
}
@Override
public long findWeixinCount(Date start, Date end,String pt) {
return dataDao.findWeixinCount(start, end,pt);
}
@Override
public int findBySourceFromLibrary(Date startTime, Date endTime, String source) { public int findBySourceFromLibrary(Date startTime, Date endTime, String source) {
return dataDao.findBySourceFromLibrary(startTime, endTime, source); return dataDao.findBySourceFromLibrary(startTime, endTime, source);
} }
...@@ -89,16 +81,19 @@ public class DataServiceImpl implements DataService{ ...@@ -89,16 +81,19 @@ public class DataServiceImpl implements DataService{
} }
@Override @Override
public long finCountByDayData(Date startTime, Date endTime, String pt, String id) { public long finCountByDayData(Date startTime, Date endTime, Template temp) {
return dataDao.finCountByDayData(startTime, endTime, pt, id); return dataDao.finCountByDayData(startTime, endTime, temp);
} }
@Override @Override
public int findByTempName(Date startTime, Date endTime, String tempName) { public int findByTempName(Date startTime, Date endTime, String tempName) {
String spId=crd.getSpIdByConfigName(tempName); String spId = crd.getSpIdByConfigName(tempName);
return dataDao.findBySpId(startTime, endTime, spId); return dataDao.findBySpId(startTime, endTime, spId);
} }
@Override
public long finCountByDayData(Date startDate, Date endDate, String pt) {
return dataDao.finCountByDayData(startDate, endDate, pt);
}
} }
...@@ -28,17 +28,9 @@ import com.mongodb.ServerAddress; ...@@ -28,17 +28,9 @@ import com.mongodb.ServerAddress;
public class MongoConnect { public class MongoConnect {
private static final Logger log = LoggerFactory.getLogger(MongoConnect.class); private static final Logger log = LoggerFactory.getLogger(MongoConnect.class);
private static MongoClient mongoClient = null; private static MongoClient mongoClient = null;
private static String host;
private static int port;
private static String username;
private static String password;
private static String dbName; private static String dbName;
public MongoConnect(String host, int port, String username, String password, String dbName) { public MongoConnect(String host, int port, String username, String password, String dbName) {
this.host = host;
this.port = port ;
this.username = username;
this.password = password;
this.dbName = dbName; this.dbName = dbName;
if (mongoClient == null) { if (mongoClient == null) {
MongoClientOptions.Builder optionsBuilder = new MongoClientOptions.Builder(); MongoClientOptions.Builder optionsBuilder = new MongoClientOptions.Builder();
...@@ -46,7 +38,6 @@ public class MongoConnect { ...@@ -46,7 +38,6 @@ public class MongoConnect {
optionsBuilder.connectTimeout(1000 * 60 * 40); // 与数据库建立连接的超时时间 optionsBuilder.connectTimeout(1000 * 60 * 40); // 与数据库建立连接的超时时间
optionsBuilder.maxWaitTime(1000 * 60 * 5); // 一个线程成功获取到一个可用数据库连接之前的最大等待时间 optionsBuilder.maxWaitTime(1000 * 60 * 5); // 一个线程成功获取到一个可用数据库连接之前的最大等待时间
optionsBuilder.threadsAllowedToBlockForConnectionMultiplier(100);// 等待一个连接 optionsBuilder.threadsAllowedToBlockForConnectionMultiplier(100);// 等待一个连接
// 允许阻塞的线程数
optionsBuilder.maxConnectionIdleTime(0); optionsBuilder.maxConnectionIdleTime(0);
optionsBuilder.maxConnectionLifeTime(0); optionsBuilder.maxConnectionLifeTime(0);
optionsBuilder.socketKeepAlive(true); optionsBuilder.socketKeepAlive(true);
...@@ -67,8 +58,6 @@ public class MongoConnect { ...@@ -67,8 +58,6 @@ public class MongoConnect {
public MongoConnect(String host, int port, String dbName) { public MongoConnect(String host, int port, String dbName) {
this.host = host;
this.port = port ;
this.dbName = dbName; this.dbName = dbName;
if (mongoClient == null) { if (mongoClient == null) {
MongoClientOptions.Builder optionsBuilder = new MongoClientOptions.Builder(); MongoClientOptions.Builder optionsBuilder = new MongoClientOptions.Builder();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment