Skip to content

D
datamanage
Commit f0e1ed56 by shenjinzhu
Options

模板检测修改

parent 6d9ea76b
Showing with 22 additions and 100 deletions
src/main/java/com/zhiwei/manage/dao/DataDao.java
......@@ -6,6 +6,7 @@ import java.util.List;
import com.zhiwei.manage.bean.Data;
import com.zhiwei.manage.bean.EasyNews;
import com.zhiwei.manage.bean.News;
import com.zhiwei.manage.bean.Template;
public interface DataDao {
......@@ -19,8 +20,8 @@ public interface DataDao {
public List<Data> findByMethod(Date startTime,Date endTime,String method,Object value);
public int findBySourceFromLibrary(Date startTime, Date endTime, String source);
public int findBySpId(Date startTime, Date endTime, String syId);
/**
* 根据数据来源来搜索数据
......@@ -31,9 +32,8 @@ public interface DataDao {
*/
public List<EasyNews> findByAccurateSource(String startTime, String endTime, String source);
public int findByAccurateType(Date startTime, Date endTime, String type);
/**
* 根据可视化模板的id去搜索数据
* @param startTime
......@@ -43,25 +43,13 @@ public interface DataDao {
*/
public List<EasyNews> findByAccurateSpId(String startTime, String endTime, String SpId);
/**
* 微博数据统计
* @param startTime
* @param endTime
* @return
*/
public long findWeiboCount(Date startTime,Date endTime);
public long findWeixinCount(Date startTime,Date endTime,String pt);
public List<String> findSourceByLikeParam(String param);
public List<String> findTmpNameByLikeParam(String param);
public List<String> findTypeByLikeParam(String param);
public long finCountByDayData(Date startTime,Date endTime,String pt,String id);
public long finCountByDayData(Date startTime,Date endTime,Template temp);
public long finCountByDayData(Date startTime,Date endTime,String pt);
}
src/main/java/com/zhiwei/manage/handle/DelayedThread.java
......@@ -45,7 +45,6 @@ public class DelayedThread extends Thread {
queue.offer(run);
cal.add(Calendar.HOUR, run.getDelayedInterval());
run.setNextRun(cal.getTime());
System.out.println("开始");
}
}
Tools.sleep(1000);
......
src/main/java/com/zhiwei/manage/handle/MainThread.java
......@@ -46,7 +46,6 @@ public class MainThread extends Thread {
/**
* 加入执行map
*
* @param template
*/
public static boolean pushMap(String templateId) {
......@@ -148,7 +147,6 @@ public class MainThread extends Thread {
ServerServiceImpl serverService = BeanFactory.getBean(ServerServiceImpl.class);
MessageServiceImpl msgService = BeanFactory.getBean(MessageServiceImpl.class);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
// int num = 1;
while (true) {
Calendar countTest = Calendar.getInstance();
Calendar start = Calendar.getInstance();
......@@ -161,8 +159,6 @@ public class MainThread extends Thread {
} catch (ParseException e1) {
e1.printStackTrace();
}
// int num = 1;
// for (int i = 0; i < 19; i++) {
start.add(Calendar.DATE, -1);
countTest.add(Calendar.DATE, -7);
try {
......@@ -171,53 +167,15 @@ public class MainThread extends Thread {
List<Channel> channel = serverService.findDisChannel();
for (Channel chan : channel) {
try {
if (chan.getPt().equals("网媒")) {
List<CrawTemplate> list = crawService.findAll();
// 每日的数据量入库（对应模板）
for (CrawTemplate cr : list) {
count = (int) dataService.finCountByDayData(start.getTime(), end.getTime(),
cr.getPt(), cr.getId());
Data data = new Data();
data.setCount(count);
data.setTempName(cr.getConfigName());
data.setTime(start.getTime());
data.setSpyderInfoId(cr.getId());
dataService.insert(data);
if (count == 0) {
int testCount = (int) dataService.finCountByDayData(countTest.getTime(),
end.getTime(), cr.getPt(), cr.getId());
if (testCount == 0) {
Message m = new Message();
m.setHandle(false);
m.setCreateDate(start.getTime());
m.setTemplateLv(1);
m.setTempName(cr.getConfigName());
m.setPt(cr.getPt());
m.setErrorMsg("数据为0检测：" + cr.getConfigName() + "," + cr.getId() + "数据为0");
msgService.insert(m);
}
}
log.info(TimeUtil.yearToDay(start.getTime()) + "|" + data.getTempName() + "|入库");
}
count = (int) dataService.finCountByDayData(start.getTime(), end.getTime(),
chan.getPt(), null);
Data data = new Data();
data.setCount(count);
data.setPt(chan.getPt());
data.setTime(start.getTime());
dataService.insert(data);
} else {
count = (int) dataService.finCountByDayData(start.getTime(), end.getTime(),
chan.getPt(), null);
count = (int) dataService.finCountByDayData(start.getTime(), end.getTime(), chan.getPt());
Data data = new Data();
data.setCount(count);
data.setPt(chan.getPt());
data.setTime(start.getTime());
dataService.insert(data);
log.info(TimeUtil.yearToDay(start.getTime()) + "|" + chan.getPt() + "|入库");
}
} catch (Exception e) {
log.error(chan.getPt()+"入库出错，错误信息{}",e);
}
}
Set<String> keys = allTmp.keySet();
......@@ -227,8 +185,7 @@ public class MainThread extends Thread {
if (tmp.getSpyderInfoId() != null) {
int cwCount = 0;
String pt = crawService.findPt(tmp.getSpyderInfoId());
cwCount = (int) dataService.finCountByDayData(start.getTime(), end.getTime(), tmp.getPt(),
tmp.getSpyderInfoId());
cwCount = (int) dataService.finCountByDayData(start.getTime(), end.getTime(), tmp);
if (cwCount < tmp.getNewsNumber()) {
Message m = new Message();
m.setHandle(false);
......@@ -246,9 +203,6 @@ public class MainThread extends Thread {
} finally {
mainLock.unlock();
}
// end.add(Calendar.DATE, -1);
// num++;
// }
Tools.sleep(1000 * 60 * 60);
}
Tools.sleep(1000 * 30);
......
src/main/java/com/zhiwei/manage/handle/PingThread.java
......@@ -46,7 +46,6 @@ public class PingThread extends Thread {
PingUrl run = runMap.get(key);
Calendar cal = Calendar.getInstance();
if (run.getNextRun().before(cal.getTime())) {
System.out.println("ping添加一个");
queue.offer(run);
cal.add(Calendar.HOUR, run.getPingInterval());
run.setNextRun(cal.getTime());
......
src/main/java/com/zhiwei/manage/handle/Task.java
......@@ -50,7 +50,7 @@ public class Task implements Runnable {
Template tp = MainThread.mainMap.get(ping.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg(code == 0 ? ping.getPingUrlOne() + "访问失败,错误代码" + code : "访问超时");
msg.setErrorMsg(code == 0 ? "访问超时":ping.getPingUrlOne() + "访问失败,错误代码" + code );
msg.setHandle(false);
msg.setTemplateCreator(tp.getCreator());
msg.setTempName(ping.getTempName());
......
src/main/java/com/zhiwei/manage/service/DataService.java
......@@ -5,6 +5,7 @@ import java.util.List;
import com.zhiwei.manage.bean.Data;
import com.zhiwei.manage.bean.EasyNews;
import com.zhiwei.manage.bean.Template;
public interface DataService {
......@@ -28,15 +29,12 @@ public interface DataService {
public int findByTempName(Date startTime, Date endTime, String tempName);
public long findWeiboCount(Date start, Date end);
public long findWeixinCount(Date start, Date end,String pt);
public List<String> findSourceByLikeParam(String param);
public List<String> findTmpNameByLikeParam(String param);
public List<String> findTypeByLikeParam(String param);
//根据配置获取数据
public long finCountByDayData(Date startDate,Date endDate,String pt,String id);
public long finCountByDayData(Date startDate,Date endDate,Template temp);
public long finCountByDayData(Date startDate,Date endDate,String pt);
}
src/main/java/com/zhiwei/manage/service/DataServiceImpl.java
......@@ -8,10 +8,12 @@ import org.springframework.stereotype.Service;
import com.zhiwei.manage.bean.Data;
import com.zhiwei.manage.bean.EasyNews;
import com.zhiwei.manage.bean.Template;
import com.zhiwei.manage.dao.CrawTemplateDao;
import com.zhiwei.manage.dao.DataDao;
@Service
public class DataServiceImpl implements DataService{
public class DataServiceImpl implements DataService {
@Autowired
private DataDao dataDao;
......@@ -49,16 +51,6 @@ public class DataServiceImpl implements DataService{
}
@Override
public long findWeiboCount(Date start, Date end) {
return dataDao.findWeiboCount(start, end);
}
@Override
public long findWeixinCount(Date start, Date end,String pt) {
return dataDao.findWeixinCount(start, end,pt);
}
@Override
public int findBySourceFromLibrary(Date startTime, Date endTime, String source) {
return dataDao.findBySourceFromLibrary(startTime, endTime, source);
}
......@@ -89,16 +81,19 @@ public class DataServiceImpl implements DataService{
}
@Override
public long finCountByDayData(Date startTime, Date endTime, String pt, String id) {
return dataDao.finCountByDayData(startTime, endTime, pt, id);
public long finCountByDayData(Date startTime, Date endTime, Template temp) {
return dataDao.finCountByDayData(startTime, endTime, temp);
}
@Override
public int findByTempName(Date startTime, Date endTime, String tempName) {
String spId=crd.getSpIdByConfigName(tempName);
String spId = crd.getSpIdByConfigName(tempName);
return dataDao.findBySpId(startTime, endTime, spId);
}
@Override
public long finCountByDayData(Date startDate, Date endDate, String pt) {
return dataDao.finCountByDayData(startDate, endDate, pt);
}
}
src/main/java/com/zhiwei/manage/util/MongoConnect.java
......@@ -28,17 +28,9 @@ import com.mongodb.ServerAddress;
public class MongoConnect {
private static final Logger log = LoggerFactory.getLogger(MongoConnect.class);
private static MongoClient mongoClient = null;
private static String host;
private static int port;
private static String username;
private static String password;
private static String dbName;
public MongoConnect(String host, int port, String username, String password, String dbName) {
this.host = host;
this.port = port ;
this.username = username;
this.password = password;
this.dbName = dbName;
if (mongoClient == null) {
MongoClientOptions.Builder optionsBuilder = new MongoClientOptions.Builder();
......@@ -46,7 +38,6 @@ public class MongoConnect {
optionsBuilder.connectTimeout(1000 * 60 * 40); // 与数据库建立连接的超时时间
optionsBuilder.maxWaitTime(1000 * 60 * 5); // 一个线程成功获取到一个可用数据库连接之前的最大等待时间
optionsBuilder.threadsAllowedToBlockForConnectionMultiplier(100);// 等待一个连接
// 允许阻塞的线程数
optionsBuilder.maxConnectionIdleTime(0);
optionsBuilder.maxConnectionLifeTime(0);
optionsBuilder.socketKeepAlive(true);
......@@ -67,8 +58,6 @@ public class MongoConnect {
public MongoConnect(String host, int port, String dbName) {
this.host = host;
this.port = port ;
this.dbName = dbName;
if (mongoClient == null) {
MongoClientOptions.Builder optionsBuilder = new MongoClientOptions.Builder();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment