Skip to content

M
messageflow
Commit ed96a533 by shentao
Options

2018/11/19 稳定版：追踪规则+消息流灵活上限1.0.0版

parent 22a3d9e4
Showing with 1680 additions and 938 deletions
src/main/java/com/zhiwei/messageflow/ES4RedisStart.java
......@@ -30,11 +30,15 @@ public class ES4RedisStart {
@Autowired
private ES4RedisTask es4RedisTask;
@Autowired
private ESGetCommonId esGetCommonId;
/**
* 启动线程
*/
public void startThread() throws JsonParseException, JsonMappingException, IOException, InterruptedException {
//更新commonid
esGetCommonId.getCommonId();
// 项目列表
List<Project> projects = projectDao.getAllProjects();
......@@ -44,7 +48,7 @@ public class ES4RedisStart {
// 遍历项目
for (Project project : projects) {
// if(!project.getProjectName().equals("今日头条")) {
// if(!project.getProjectName().equals("测试")) {
// continue;
// }
......
src/main/java/com/zhiwei/messageflow/ES4RedisTask.java
......@@ -97,29 +97,6 @@ public class ES4RedisTask {
// 项目预警规则列表
List<TrackRule> trackRules = trackRuleDao.getTrackRuleByProject(project.getProjectName());
/*
* //测试预警规则 List<TrackRule> trackRules=new ArrayList<>(); TrackRule tr1=new
* TrackRule(); TrackRule tr2=new TrackRule(); TrackRule tr3=new TrackRule();
* Date date = new Date(); Long createAt=date.getTime();
* tr1.setCreateAt(createAt); tr1.setAndOr(""); tr1.setWarn(false);
* tr1.setRuleType("keyWords"); tr1.setSubmitter("虞诚毅");
* tr1.setKeyWordsInputOne("知乎"); tr1.setRuleName("测试规则1");
* tr1.setRuleExplain("测试说明1"); tr1.setEarlyWarningTime("三天");
* tr1.setProject("腾讯"); tr1.setEarlyWarning("email");
* tr2.setCreateAt(createAt); tr2.setAndOr("或"); tr2.setWarn(false);
* tr2.setRuleType("keyWords"); tr2.setSubmitter("虞诚毅");
* tr2.setKeyWordsInputOne("知乎"); tr2.setKeyWordsInputTwo("微博");
* tr2.setRuleName("测试规则2"); tr2.setRuleExplain("测试说明3");
* tr2.setEarlyWarningTime("三天"); tr2.setProject("腾讯");
* tr2.setEarlyWarning("email"); tr3.setCreateAt(createAt); tr3.setAndOr("且");
* tr3.setWarn(false); tr3.setRuleType("keyWords"); tr3.setSubmitter("虞诚毅");
* tr3.setKeyWordsInputOne("知乎"); tr3.setKeyWordsInputTwo("问题");
* tr3.setRuleName("测试规则2"); tr3.setRuleExplain("测试说明3");
* tr3.setEarlyWarningTime("三天"); tr3.setProject("腾讯");
* tr3.setEarlyWarning("wechat"); trackRules.add(tr1); trackRules.add(tr2);
* trackRules.add(tr3);
*/
// 遍历平台
for (PlatformNew platform : platformNames) {
......@@ -242,7 +219,7 @@ public class ES4RedisTask {
} // 遍历关键词组
// 预警
earlyWarningService.earlyWarningNew(trackRules, count, allstartrsid, allrsid, platform,
earlyWarningService.earlyWarningNew(messages,trackRules, count, allstartrsid, allrsid, platform,
project.getProjectName());
// 向redis库中存储新的rsid Map,覆盖原有数据
......
src/main/java/com/zhiwei/messageflow/ESGetCommonId.java
......@@ -2,6 +2,7 @@ package com.zhiwei.messageflow;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -19,26 +20,27 @@ public class ESGetCommonId {
@Autowired
private EventService eventService;
public static int START_COMMONID;
public static long START_COMMONID;
public static long TIME;
public void getCommonId() {
long nowtime = System.currentTimeMillis();
if (TIME < nowtime - 60 * 1000L) {
if (TIME < nowtime - 60 * 60 * 1000L) {
List<Integer> cidList = new ArrayList<>();
int weibo = esDao.get24hoursFirstCommonid("weibo");
int weibo = esDao.get48hoursFirstCommonid("weibo");
cidList.add(weibo);
int media = esDao.get24hoursFirstCommonid("media");
int media = esDao.get48hoursFirstCommonid("media");
cidList.add(media);
int video = esDao.get24hoursFirstCommonid("video");
int video = esDao.get48hoursFirstCommonid("video");
cidList.add(video);
int zhihu = esDao.get24hoursFirstCommonid("zhihu");
int zhihu = esDao.get48hoursFirstCommonid("zhihu");
cidList.add(zhihu);
Collections.sort(cidList);
System.err.println(weibo + " " + media + " " + video + " " + zhihu + "\n" + cidList);
START_COMMONID = cidList.get(0);
TIME = nowtime;
System.err.println("START_COMMONID:" + START_COMMONID + "\tNOW:" + new Date(TIME));
}
}
......@@ -55,7 +57,8 @@ public class ESGetCommonId {
int zhihu = esDao.getCommonidByTime("zhihu", endtime);
cidList.add(zhihu);
Collections.sort(cidList);
// System.err.println(weibo + " " + media + " " + video + " " + zhihu + "\n" + cidList);
// System.err.println(weibo + " " + media + " " + video + " " + zhihu + "\n" +
// cidList);
int commonid = cidList.get(0);
return commonid;
}
......
src/main/java/com/zhiwei/messageflow/config/RedisConfig.java
......@@ -31,6 +31,8 @@ public class RedisConfig {
public static final String EVENTLISTKEY = "Event:EventList";
/** 事件采集去重url集 **/
public static final String EVENTHASHKEY = "Event:Hash:";
/** 追踪规则已追踪集 **/
public static final String TRACKKEY = "TrackRule:";
private int keyMaxSize;
......
src/main/java/com/zhiwei/messageflow/es/dao/ESDao.java
......@@ -101,7 +101,7 @@ public interface ESDao {
public SearchHits getDataFromEs(List<String> keywords, int count, long startid, long endid, PlatformNew platform,
String project, String matchFields);
public int get24hoursFirstCommonid(String tableName);
public int get48hoursFirstCommonid(String tableName);
public int getCommonidByTime(String tableName, long endtime);
/**
......
src/main/java/com/zhiwei/messageflow/es/dao/impl/ESDaoImpl.java
......@@ -725,14 +725,17 @@ public class ESDaoImpl implements ESDao {
}
@Override
public int get24hoursFirstCommonid(String tableName) {
public int get48hoursFirstCommonid(String tableName) {
// 获取当天最早commonid
int commonid = 0;
long date = System.currentTimeMillis() - 1 * 24 * 60 * 60 * 1000L;
String weibotime = TimeUtil.formatDateToMinute(new Date(date));
String zhihutime = TimeUtil.formatEsDate(new Date(date - 8 * 3600 * 1000L)) + "Z";
String vmtime = TimeUtil.formatEsDate(new Date(date - 8 * 3600 * 1000L));
long now =System.currentTimeMillis();
long date = now - 1 * 48 * 60 * 60 * 1000L;
// String wEtime = TimeUtil.formatDateToMinute(new Date(now));
// String weibotime = TimeUtil.formatDateToMinute(new Date(date));
// String zEtime = TimeUtil.formatEsDate(new Date(now - 8 * 3600 * 1000L)) + "Z";
// String zhihutime = TimeUtil.formatEsDate(new Date(date - 8 * 3600 * 1000L)) + "Z";
// String vmEtime = TimeUtil.formatEsDate(new Date(now - 8 * 3600 * 1000L));
// String vmtime = TimeUtil.formatEsDate(new Date(date - 8 * 3600 * 1000L));
// weibo
if (tableName.equals("weibo")) {
List<String> list = indexes.getLastIndexes("network", 2);
......@@ -741,7 +744,7 @@ public class ESDaoImpl implements ESDao {
.setQuery(QueryBuilders.matchAllQuery()).setFetchSource(new String[] { "commonid" }, null)
.addSort("commonid", SortOrder.ASC).setFrom(0).setSize(1);
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("time").from(weibotime));
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("savetime").from(date).to(now));
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
......@@ -754,7 +757,7 @@ public class ESDaoImpl implements ESDao {
.setQuery(QueryBuilders.matchAllQuery()).setFetchSource(new String[] { "commonid" }, null)
.addSort("commonid", SortOrder.ASC).setFrom(0).setSize(1);
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("time").from(vmtime));
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("savetime").from(date).to(now));
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
......@@ -767,7 +770,7 @@ public class ESDaoImpl implements ESDao {
.setQuery(QueryBuilders.matchAllQuery()).setFetchSource(new String[] { "commonid" }, null)
.addSort("commonid", SortOrder.ASC).setFrom(0).setSize(1);
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("time").from(vmtime));
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("savetime").from(date).to(now));
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
......@@ -778,7 +781,7 @@ public class ESDaoImpl implements ESDao {
.setQuery(QueryBuilders.matchAllQuery()).setFetchSource(new String[] { "commonid" }, null)
.addSort("commonid", SortOrder.ASC).setFrom(0).setSize(1);
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("insert_at").from(zhihutime));
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("savetime").from(date).to(now));
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
......
src/main/java/com/zhiwei/messageflow/es/service/EarlyWarningService.java
......@@ -2,6 +2,7 @@ package com.zhiwei.messageflow.es.service;
import java.util.List;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.messageflow.mongo.bean.PlatformNew;
import com.zhiwei.messageflow.mongo.bean.TrackRule;
......@@ -9,6 +10,7 @@ public interface EarlyWarningService {
/**
* 预警
* @param messages
*
* @param trackRules
* 预警规则
......@@ -23,9 +25,6 @@ public interface EarlyWarningService {
* @param projectName
* 项目名
*/
void earlyWarning(List<TrackRule> trackRules, int count, long startid, long endid, String platformName,
String projectName);
void earlyWarningNew(List<TrackRule> trackRules, int count, long startid, Long endid, PlatformNew platform,
void earlyWarningNew(List<JSONObject> messages, List<TrackRule> trackRules, int count, long startid, Long endid, PlatformNew platform,
String projectName);
}
src/main/java/com/zhiwei/messageflow/es/service/TrackHitAndWarnService.java
package com.zhiwei.messageflow.es.service;
import org.elasticsearch.search.SearchHits;
import com.zhiwei.messageflow.mongo.bean.PlatformNew;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.messageflow.mongo.bean.TrackRule;
public interface TrackHitAndWarnService {
/**
* 关键词预警数据获取
*
* @param truckRule
* @param count
* @param startid
* @param endid
* @param platformName
* @return
*/
SearchHits keyWordsTrackHit(TrackRule truckRule, int count, long startid, long endid, String platformName);
/**
* 渠道预警数据获取
*
* @param truckRule
* @param count
* @param startid
* @param endid
* @param platformName
* @return
* 判断该条追踪规则是否处于预警状态
* @Title: isWarnTrackrule
* @Description: 判断该条追踪规则是否处于预警状态
* @param @param tr
* @param @return 设定文件
* @return boolean 返回类型
*/
SearchHits channelTrackHit(TrackRule truckRule, int count, long startid, long endid, String platformName);
boolean isWarnTrackrule(TrackRule trackRule);
/**
* 相似新闻数预警数据获取
*
* @param truckRule
* @param count
* @param startid
* @param endid
* @param platformName
* @return
* 使用单条追踪规则
* @Title: useTrackrule
* @Description: 使用单条追踪规则
* @param @param trackrule
* @param @return 设定文件
* @return boolean 返回类型
*/
SearchHits articleTrackHit(TrackRule truckRule, int count, long startid, long endid, String platformName);
boolean endTrackrule(TrackRule trackrule);
/**
* 邮件预警
*
* @param trackHits
* 是否命中预警规则
* @param platformName
* @param trackRule
* @Title: ishitWarnMsg
* @Description:是否命中预警规则
* @param @param msg
* @param @param tr 设定文件
* @return void 返回类型
*/
void WarnEmail(SearchHits trackHit, String platformName, TrackRule trackRule, String projectName);
boolean ishitWarnMsg(JSONObject msg, String platformName, TrackRule tr);
/**
* 微信预警
*
* @param trackHits
* @param platformName
* @param trackRule
* 按不同规则预警
* @Title: warnMsg
* @Description: 按不同规则预警
* @param @param msg
* @param @param tr 设定文件
* @return void 返回类型
*/
void WarnWechat(SearchHits trackHit, String platformName, TrackRule trackRule, String projectName);
SearchHits keyWordsTrackHitNew(TrackRule trackRule, int count, long startid, Long endid, PlatformNew platform);
SearchHits channelTrackHitNew(TrackRule trackRule, int count, long startid, Long endid, PlatformNew platform);
SearchHits articleTrackHitNew(TrackRule trackRule, int count, long startid, Long endid, PlatformNew platform);
void WarnWechatNew(SearchHits trackHit, String platformName, TrackRule trackRule, String projectName);
void WarnEmailNew(SearchHits trackHit, String platformName, TrackRule trackRule, String projectName);
void warnMsg(JSONObject msg, TrackRule tr);
}
src/main/java/com/zhiwei/messageflow/es/service/impl/EarlyWarningServiceImpl.java
......@@ -2,176 +2,37 @@ package com.zhiwei.messageflow.es.service.impl;
import java.util.List;
import org.elasticsearch.search.SearchHits;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.messageflow.es.service.EarlyWarningService;
import com.zhiwei.messageflow.es.service.TrackHitAndWarnService;
import com.zhiwei.messageflow.mongo.bean.PlatformNew;
import com.zhiwei.messageflow.mongo.bean.TrackRule;
import com.zhiwei.messageflow.mongo.dao.TrackRuleDao;
@Component
public class EarlyWarningServiceImpl implements EarlyWarningService {
@Autowired
private TrackRuleDao trackRuleDao;
@Autowired
private TrackHitAndWarnService trackHitAndWarnService;
@Override
public void earlyWarning(List<TrackRule> trackRules, int count, long startid, long endid, String platformName,
String projectName) {
for (TrackRule trackRule : trackRules) {
// 判断是否预警
if (trackRule.isWarn()) {
continue;
}
/**
* 判断是否过期
*/
long createAt = trackRule.getCreateAt();
long addDay = 0;
// if (trackRule.getEarlyWarningTime().equals("一天")) {
// addDay = 24 * 60 * 60 * 1000L;
// } else if (trackRule.getEarlyWarningTime().equals("二天")) {
// addDay = 2 * 24 * 60 * 60 * 1000L;
// } else if (trackRule.getEarlyWarningTime().equals("三天")) {
// addDay = 3 * 24 * 60 * 60 * 1000L;
// }
long confirm = System.currentTimeMillis()+1L;//createAt + addDay;
if (confirm <= System.currentTimeMillis()) {
// 过期
trackRule.setWarn(true);
trackRuleDao.updateTrackrule(trackRule.get_id());
} else {
// 判断规则
// 首先判断预警方式
if (trackRule.getEarlyWarning().equals("no")) {
} else if (trackRule.getEarlyWarning().equals("wechat")) {
// 微信预警
// 处理规则
SearchHits TrackHit = null;
if (trackRule.getRuleType().equals("keyWords")) {
// 关键词追踪
TrackHit = trackHitAndWarnService.keyWordsTrackHit(trackRule, count, startid, endid,
platformName);
} else if (trackRule.getRuleType().equals("channel")) {
// 渠道追踪
TrackHit = trackHitAndWarnService.channelTrackHit(trackRule, count, startid, endid,
platformName);
} else if (trackRule.getRuleType().equals("article")) {
// 相似新闻数追踪
TrackHit = trackHitAndWarnService.articleTrackHit(trackRule, count, startid, endid,
platformName);
}
trackHitAndWarnService.WarnWechat(TrackHit, platformName, trackRule, projectName);
} else if (trackRule.getEarlyWarning().equals("email")) {
// 邮箱预警
// 处理规则
SearchHits TrackHit = null;
if (trackRule.getRuleType().equals("keyWords")) {
// 关键词追踪
TrackHit = trackHitAndWarnService.keyWordsTrackHit(trackRule, count, startid, endid,
platformName);
} else if (trackRule.getRuleType().equals("channel")) {
// 渠道追踪
TrackHit = trackHitAndWarnService.channelTrackHit(trackRule, count, startid, endid,
platformName);
} else if (trackRule.getRuleType().equals("article")) {
// 相似新闻数追踪
TrackHit = trackHitAndWarnService.articleTrackHit(trackRule, count, startid, endid,
platformName);
}
trackHitAndWarnService.WarnEmail(TrackHit, platformName, trackRule, projectName);
}
}
}
}
@Override
public void earlyWarningNew(List<TrackRule> trackRules, int count, long startid, Long endid, PlatformNew platform,
String projectName) {
public void earlyWarningNew(List<JSONObject> messages, List<TrackRule> trackRules, int count, long startid,
Long endid, PlatformNew platform, String projectName) {
String platformName = platform.getPlatformName();
for (TrackRule trackRule : trackRules) {
for (int i = 0; i < messages.size(); i++) {
JSONObject msg = messages.get(i);
for (TrackRule tr : trackRules) {
// 判断是否预警
if (trackRule.isWarn()) {
if (!trackHitAndWarnService.isWarnTrackrule(tr))
continue;
}
/**
* 判断是否过期
*/
long createAt = trackRule.getCreateAt();
long addDay = 0;
// if (trackRule.getEarlyWarningTime().equals("一天")) {
// addDay = 24 * 60 * 60 * 1000L;
// } else if (trackRule.getEarlyWarningTime().equals("二天")) {
// addDay = 2 * 24 * 60 * 60 * 1000L;
// } else if (trackRule.getEarlyWarningTime().equals("三天")) {
// addDay = 3 * 24 * 60 * 60 * 1000L;
// } else if (trackRule.getEarlyWarningTime().equals("一月")) {
// addDay = 30 * 24 * 60 * 60 * 1000L;
// }
long confirm = System.currentTimeMillis()+1L;//createAt + addDay;
if (confirm <= System.currentTimeMillis()) {
// 过期
trackRule.setWarn(true);
trackRuleDao.updateTrackrule(trackRule.get_id());
} else {
// 判断规则
// 首先判断预警方式
if (trackRule.getEarlyWarning().equals("no")) {
} else if (trackRule.getEarlyWarning().equals("wechat")) {
// 微信预警
// 处理规则
SearchHits TrackHit = null;
if (trackRule.getRuleType().equals("keyWords")) {
// 关键词追踪
TrackHit = trackHitAndWarnService.keyWordsTrackHitNew(trackRule, count, startid, endid,
platform);
} else if (trackRule.getRuleType().equals("channel")) {
// 渠道追踪
TrackHit = trackHitAndWarnService.channelTrackHitNew(trackRule, count, startid, endid,
platform);
} else if (trackRule.getRuleType().equals("article")) {
// 相似新闻数追踪
TrackHit = trackHitAndWarnService.articleTrackHitNew(trackRule, count, startid, endid,
platform);
}
trackHitAndWarnService.WarnWechatNew(TrackHit,platformName, trackRule, projectName);
} else if (trackRule.getEarlyWarning().equals("email")) {
// 邮箱预警
// 处理规则
SearchHits TrackHit = null;
if (trackRule.getRuleType().equals("keyWords")) {
// 关键词追踪
TrackHit = trackHitAndWarnService.keyWordsTrackHitNew(trackRule, count, startid, endid,
platform);
} else if (trackRule.getRuleType().equals("channel")) {
// 渠道追踪
TrackHit = trackHitAndWarnService.channelTrackHitNew(trackRule, count, startid, endid,
platform);
} else if (trackRule.getRuleType().equals("article")) {
// 相似新闻数追踪
TrackHit = trackHitAndWarnService.articleTrackHitNew(trackRule, count, startid, endid,
platform);
}
trackHitAndWarnService.WarnEmailNew(TrackHit,platformName, trackRule, projectName);
}
// 是否命中预警规则
boolean ishit = trackHitAndWarnService.ishitWarnMsg(msg, platformName, tr);
// 按不同预警规则预警
if (ishit)
trackHitAndWarnService.warnMsg(msg, tr);
}
}
......
src/main/java/com/zhiwei/messageflow/es/service/impl/HighLightFillingServiceImpl.java
......@@ -420,7 +420,6 @@ public class HighLightFillingServiceImpl implements HighLightFillingService {
message = mapper.writeValueAsString(sourceHitMap);
res = JSONObject.parseObject(message);
} catch (JsonProcessingException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
String type = searchHit.getType();
......
src/main/java/com/zhiwei/messageflow/es/service/impl/NoiseProcessingServiceImpl.java
......@@ -3,13 +3,9 @@ package com.zhiwei.messageflow.es.service.impl;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.TimeZone;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.search.SearchHit;
......@@ -18,7 +14,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.messageflow.DirectES4RedisThread;
import com.zhiwei.messageflow.bean.MediaMessage;
import com.zhiwei.messageflow.bean.VideoMessage;
import com.zhiwei.messageflow.bean.WeiboMessage;
......@@ -28,7 +23,6 @@ import com.zhiwei.messageflow.es.service.NoiseProcessingService;
import com.zhiwei.messageflow.mongo.bean.NoiseRule;
import com.zhiwei.messageflow.mongo.bean.PlatformNew;
import com.zhiwei.messageflow.mongo.dao.NoiseRuleDao;
import com.zhiwei.messageflow.util.ESQueryUtil;
@Component
public class NoiseProcessingServiceImpl implements NoiseProcessingService {
......@@ -651,7 +645,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
try {
date = dateFormat.parse(time);
if ((date.getTime() + (8 * 3600 * 1000L)) < zero) {
log.info(date.getTime() + (8 * 3600 * 1000L)+"|"+zero);
// log.info(date.getTime() + (8 * 3600 * 1000L)+"|"+zero);
return true;
}
} catch (ParseException e) {
......@@ -681,7 +675,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
String time = map.get("time") != null ? map.get("time").toString() : null;
if (isTimeOver(time)) {
log.info("weibo:time:" + time +"\t"+ map.get("id").toString());
// log.info("weibo:time:" + time +"\t"+ map.get("id").toString());
return true;
}
String text = map.get("text") != null ? map.get("text").toString() : null;
......@@ -786,7 +780,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
String platform = platformNew.getPlatformName();
String time = map.get("created_at") != null ? map.get("created_at").toString() : null;
if (isTimeOver(time)) {
log.info("zhihu:time:" + time +"\t"+ map.get("id").toString());
// log.info("zhihu:time:" + time +"\t"+ map.get("id").toString());
return true;
}
String text = map.get("question_title") != null ? map.get("question_title").toString() : null;
......@@ -917,7 +911,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
String platform = platformNew.getPlatformName();
String time = map.get("time") != null ? map.get("time").toString() : null;
if (isTimeOver(time)) {
log.info("video:time:" + time +"\t"+ map.get("id").toString());
// log.info("video:time:" + time +"\t"+ map.get("id").toString());
return true;
}
String text = map.get("title") != null ? map.get("title").toString() : null;
......@@ -1047,7 +1041,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
String platform = platformNew.getPlatformName();
String time = map.get("time") != null ? map.get("time").toString() : null;
if (isTimeOver(time)) {
log.info("media:time:" + time +"\t"+ map.get("id").toString());
// log.info("media:time:" + time +"\t"+ map.get("id").toString());
return true;
}
String text = map.get("title") != null ? map.get("title").toString() : null;
......
src/main/java/com/zhiwei/messageflow/es/service/impl/TrackHitAndWarnServiceImpl.java
......@@ -3,26 +3,29 @@ package com.zhiwei.messageflow.es.service.impl;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.ESGetCommonId;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.messageflow.es.bean.TemplateData;
import com.zhiwei.messageflow.es.dao.TrackESDao;
import com.zhiwei.messageflow.es.service.TrackHitAndWarnService;
import com.zhiwei.messageflow.mongo.bean.PlatformNew;
import com.zhiwei.messageflow.mongo.bean.TrackRule;
import com.zhiwei.messageflow.mongo.dao.TrackRuleDao;
import com.zhiwei.messageflow.mongo.service.UserMailService;
import com.zhiwei.messageflow.mongo.service.WechatSendService;
import com.zhiwei.messageflow.redis.service.RedisService;
import com.zhiwei.messageflow.util.TimeUtil;
import com.zhiwei.messageflow.util.Tools;
@Component
public class TrackHitAndWarnServiceImpl implements TrackHitAndWarnService {
......@@ -30,12 +33,6 @@ public class TrackHitAndWarnServiceImpl implements TrackHitAndWarnService {
private final static Logger log = LogManager.getLogger(TrackHitAndWarnServiceImpl.class);
@Autowired
private TrackESDao trackESDao;
@Autowired
private ESGetCommonId esGetCommonId;
@Autowired
TrackRuleDao trackRuleDao;
@Autowired
......@@ -44,589 +41,298 @@ public class TrackHitAndWarnServiceImpl implements TrackHitAndWarnService {
@Autowired
private WechatSendService wechatSendService;
@Override
public SearchHits keyWordsTrackHit(TrackRule truckRule, int count, long startid, long endid, String platformName) {
SearchHits searchHits = null;
String anyWord = "";
String allWords = "";
// 取关键词
if (truckRule.getAndOr().equals("或")) {
anyWord = truckRule.getKeyWordsInputOne() + "，" + truckRule.getKeyWordsInputTwo();
} else if (truckRule.getAndOr().equals("且")) {
allWords = truckRule.getKeyWordsInputOne() + "，" + truckRule.getKeyWordsInputTwo();
} else {
allWords = truckRule.getKeyWordsInputOne();
}
// 取高亮颜色
String color = "#ff330D";
if (null != truckRule.getHighlighted() && "" != truckRule.getHighlighted()) {
color = truckRule.getHighlighted();
}
// 根据平台获取数据
if (platformName.equals("微博")) {
searchHits = trackESDao.getkeyWordsTrackWeiboFromEs(anyWord, allWords, color, count, startid, endid);
} else if (platformName.equals("知乎")) {
searchHits = trackESDao.getkeyWordsTrackZhihuFromEs(anyWord, allWords, color, count, startid, endid);
} else if (platformName.equals("视频")) {
searchHits = trackESDao.getkeyWordsTrackVideoFromEs(anyWord, allWords, color, count, startid, endid);
} else {
searchHits = trackESDao.getkeyWordsTrackMediaFromEs(anyWord, allWords, color, count, startid, endid,
platformName);
}
return searchHits;
}
@Autowired
private RedisService redisService;
static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
static SimpleDateFormat sdfUS = new SimpleDateFormat("E MMM dd HH:mm:ss z yyyy", Locale.US);
/** 追踪规则已追踪集上限 **/
private static final int TRACKMAX = 100;
@Override
public SearchHits channelTrackHit(TrackRule truckRule, int count, long startid, long endid, String platformName) {
SearchHits searchHits = null;
// 取渠道平台
String pt = truckRule.getChannelPt();
String channel = truckRule.getChannelQd();
// 取高亮颜色
String color = "#ff330D";
if (null != truckRule.getHighlighted() && "" != truckRule.getHighlighted()) {
color = truckRule.getHighlighted();
}
if (pt.equals("微博")) {
searchHits = trackESDao.getchannelTrackWeiboFromEs(channel, color, count, startid, endid);
} else if (pt.equals("知乎")) {
searchHits = trackESDao.getchannelTrackZhihuFromEs(channel, color, count, startid, endid);
} else if (pt.equals("视频")) {
searchHits = trackESDao.getchannelTrackVideoFromEs(channel, color, count, startid, endid);
public boolean isWarnTrackrule(TrackRule trackrule) {
if (!trackrule.getIsUsed()) {
return false;
} else {
searchHits = trackESDao.getchannelTrackMediaFromEs(channel, color, count, startid, endid, platformName);
// 判断是否在预警区间
long startTime = trackrule.getStartTime();
long endTime = trackrule.getEndTime();
long now = System.currentTimeMillis();
if (startTime <= now && endTime >= now) {
return true;
}
return searchHits;
if (endTime < now) {
// 当前时间超过预警结束时间，自动结束
trackrule.setIsUsed(false);
trackrule.setUpdateAt(now);
trackrule.setWarnNum(0);
endTrackrule(trackrule);
return false;
}
@Override
public SearchHits articleTrackHit(TrackRule truckRule, int count, long startid, long endid, String platformName) {
SearchHits searchHits = null;
// 取相似新闻数信息
String articleTitle = truckRule.getArticleTitle();
String timeRange = truckRule.getTimeRange();
long createAt = truckRule.getCreateAt();
// 算时间区段
if (timeRange.equals("3小时内")) {
startid = createAt;
endid = createAt + 3 * 60 * 60 * 1000L;
} else if (timeRange.equals("6小时内")) {
startid = createAt;
endid = createAt + 6 * 60 * 60 * 1000L;
} else if (timeRange.equals("12小时内")) {
startid = createAt;
endid = createAt + 12 * 60 * 60 * 1000L;
} else if (timeRange.equals("24小时内")) {
startid = createAt;
endid = createAt + 24 * 60 * 60 * 1000L;
}
// 取高亮颜色
String color = "#ff330D";
if (null != truckRule.getHighlighted() && "" != truckRule.getHighlighted()) {
color = truckRule.getHighlighted();
}
if (platformName.equals("微博")) {
searchHits = trackESDao.getarticleTrackWeiboFromEs(articleTitle, color, count, startid, endid);
} else if (platformName.equals("知乎")) {
searchHits = null;
} else if (platformName.equals("视频")) {
searchHits = trackESDao.getarticleTrackVideoFromEs(articleTitle, color, count, startid, endid);
;
} else {
searchHits = trackESDao.getarticleTrackMediaFromEs(articleTitle, color, count, startid, endid,
platformName);
}
return searchHits;
return false;
}
@Override
public void WarnEmail(SearchHits trackHit, String platformName, TrackRule trackRule, String projectName) {
for (SearchHit hit : trackHit) {
String url = "";
String title = "";
String content = "";
String source = "";
String time = "";
String pt = "";
int fensi = -1;
if (platformName.equals("微博")) {
url = hit.getSource().get("url") + "";
title = "";
content = hit.getSource().get("text") + "";
source = hit.getSource().get("username") + "";
time = hit.getSource().get("time") + "";
pt = "微博";
fensi = Integer.valueOf(hit.getSource().get("fensi") + "");
} else if (platformName.equals("知乎")) {
url = hit.getId() + "";
title = hit.getSource().get("question_title") + "";
content = hit.getSource().get("answer_content") + "";
source = hit.getSource().get("user_name") + "";
time = hit.getSource().get("created_at") + "";
pt = "知乎";
} else if (platformName.equals("视频")) {
url = hit.getId() + "";
title = hit.getSource().get("title") + "";
content = "";
source = hit.getSource().get("uper") + "";
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String timein = String.valueOf(hit.getSource().get("time"));
String timeout = timein.replaceAll("T", " ");
try {
Date date = sdf.parse(timeout);
try {
// "2017-09-28 20:35:33"
Date mediadate = TimeUtil.parseTime(TimeUtil.formatTime(TimeUtil.addTimeHour(date, 8)),
"yyyy-MM-dd HH:mm:ss");
time = sdf.format(mediadate);
} catch (Exception e) {
time = sdf.format(System.currentTimeMillis());
public boolean endTrackrule(TrackRule trackrule) {
Update update = new Update();
update.set("isUsed", trackrule.getIsUsed());
update.set("updateAt", trackrule.getUpdateAt());
update.set("warnNum", trackrule.getWarnNum());
Query query = new Query();
query.addCriteria(Criteria.where("_id").is(trackrule.get_id()));
return trackRuleDao.updateTrackrule(query, update);
}
} catch (ParseException e) {
e.printStackTrace();
}
pt = "视频";
} else {
String trueid = hit.getSource().get("id") != null ? String.valueOf(hit.getSource().get("id"))
: String.valueOf(hit.getId());
url = trueid;
title = hit.getSource().get("title") + "";
content = hit.getSource().get("content") + "";
source = hit.getSource().get("source") + "";
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String timein = String.valueOf(hit.getSource().get("time"));
String timeout = timein.replaceAll("T", " ");
try {
Date date = sdf.parse(timeout);
try {
// "2017-09-28 20:35:33"
Date mediadate = TimeUtil.parseTime(TimeUtil.formatTime(TimeUtil.addTimeHour(date, 8)),
"yyyy-MM-dd HH:mm:ss");
time = sdf.format(mediadate);
} catch (Exception e) {
time = sdf.format(System.currentTimeMillis());
}
} catch (ParseException e) {
e.printStackTrace();
}
pt = platformName;
}
TemplateData templateData = new TemplateData();
if (trackRule.getRuleType().equals("keyWords")) {
// 关键词追踪
templateData.setWordRule(trackRule.getRuleExplain());
} else if (trackRule.getRuleType().equals("channel")) {
// 渠道追踪
templateData.setChannelRule(trackRule.getRuleExplain());
}
templateData.setUrl(url);
templateData.setTitle(title);
templateData.setContent(content);
templateData.setSource(source);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");
Date datetime;
try {
datetime = sdf.parse(time);
templateData.setTime(datetime);
} catch (ParseException e) {
e.printStackTrace();
templateData.setTime(new Date());
}
templateData.setPt(pt);
if (fensi != -1) {
templateData.setFensi(fensi);
}
Map<String, Object> result = new HashMap<String, Object>();
result = userMailService.sendMailByProject(projectName, templateData);
if ((result.get("message") + "").equals("用户组邮件发送成功")) {
log.info("发送邮件预警成功");
trackRule.setWarn(true);
trackRuleDao.updateTrackrule(trackRule.get_id());
@Override
public boolean ishitWarnMsg(JSONObject msg, String platformName, TrackRule tr) {
boolean isWarn = false;
switch (tr.getRuleType()) {
case "keyword":
String allText = getMsgAllText(msg);
isWarn = isKeywordWarn(allText, tr.getKeyword());
break;
} else if ((result.get("message") + "").equals("用户组邮箱未获取到")) {
log.error("发送邮件预警失败，用户获取问题");
} else {
log.error("本次部分条发送失败");
trackRule.setWarn(true);
trackRuleDao.updateTrackrule(trackRule.get_id());
case "channel":
String channelPt = platformName;
String channelName = getMsgChannelName(msg);
String channelId = getMsgChannelId(msg);
isWarn = isChannelWarn(channelPt, channelName, channelId, tr.getChannel());
break;
default:
isWarn = false;
break;
}
}
return isWarn;
}
@Override
public void WarnWechat(SearchHits trackHit, String platformName, TrackRule trackRule, String projectName) {
for (SearchHit searchHit : trackHit) {
String url = "";
String title = "";
String content = "";
String source = "";
String time = "";
String pt = "";
int fensi = -1;
if (platformName.equals("微博")) {
url = searchHit.getSource().get("url") + "";
title = "";
content = searchHit.getSource().get("text") + "";
source = searchHit.getSource().get("username") + "";
time = searchHit.getSource().get("time") + "";
pt = "微博";
fensi = Integer.valueOf(searchHit.getSource().get("fensi") + "");
} else if (platformName.equals("知乎")) {
url = searchHit.getId() + "";
title = searchHit.getSource().get("question_title") + "";
content = searchHit.getSource().get("answer_content") + "";
source = searchHit.getSource().get("user_name") + "";
time = searchHit.getSource().get("created_at") + "";
pt = "知乎";
} else if (platformName.equals("视频")) {
url = searchHit.getId() + "";
title = searchHit.getSource().get("title") + "";
content = "";
source = searchHit.getSource().get("uper") + "";
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String timein = String.valueOf(searchHit.getSource().get("time"));
String timeout = timein.replaceAll("T", " ");
try {
Date date = sdf.parse(timeout);
try {
// "2017-09-28 20:35:33"
Date mediadate = TimeUtil.parseTime(TimeUtil.formatTime(TimeUtil.addTimeHour(date, 8)),
"yyyy-MM-dd HH:mm:ss");
time = sdf.format(mediadate);
} catch (Exception e) {
time = sdf.format(System.currentTimeMillis());
}
} catch (ParseException e) {
e.printStackTrace();
}
pt = "视频";
private String getMsgChannelName(JSONObject msg) {
String res = "";
if (!Tools.isEmpty(msg.get("markPt"))) {
switch (msg.getString("markPt")) {
case "微博":
if (!Tools.isEmpty(msg.get("username")))
res = msg.getString("username");
break;
case "知乎":
if (!Tools.isEmpty(msg.get("user_name")))
res = msg.getString("user_name");
break;
case "视频":
if (!Tools.isEmpty(msg.get("pt")) && "视频".equals(msg.getString("pt"))) {
if (!Tools.isEmpty(msg.get("uper")))
res = msg.getString("uper");
} else {
String trueid = searchHit.getSource().get("id") != null
? String.valueOf(searchHit.getSource().get("id"))
: String.valueOf(searchHit.getId());
url = trueid;
title = searchHit.getSource().get("title") + "";
content = searchHit.getSource().get("content") + "";
source = searchHit.getSource().get("source") + "";
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String timein = String.valueOf(searchHit.getSource().get("time"));
String timeout = timein.replaceAll("T", " ");
try {
Date date = sdf.parse(timeout);
try {
// "2017-09-28 20:35:33"
Date mediadate = TimeUtil.parseTime(TimeUtil.formatTime(TimeUtil.addTimeHour(date, 8)),
"yyyy-MM-dd HH:mm:ss");
time = sdf.format(mediadate);
} catch (Exception e) {
time = sdf.format(System.currentTimeMillis());
if (!Tools.isEmpty(msg.get("source")))
res = msg.getString("source");
}
} catch (ParseException e) {
e.printStackTrace();
break;
case "网媒":
if (!Tools.isEmpty(msg.get("source")))
res = msg.getString("source");
break;
default:
break;
}
pt = platformName;
}
TemplateData templateData = new TemplateData();
if (trackRule.getRuleType().equals("keyWords")) {
// 关键词追踪
templateData.setWordRule(trackRule.getRuleExplain());
} else if (trackRule.getRuleType().equals("channel")) {
// 渠道追踪
templateData.setChannelRule(trackRule.getRuleExplain());
}
templateData.setUrl(url);
templateData.setTitle(title);
templateData.setContent(content);
templateData.setSource(source);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");
Date datetime;
try {
datetime = sdf.parse(time);
templateData.setTime(datetime);
} catch (ParseException e) {
e.printStackTrace();
templateData.setTime(new Date());
return res;
}
templateData.setPt(pt);
if (fensi != -1) {
templateData.setFensi(fensi);
private String getMsgChannelId(JSONObject msg) {
String res = "";
if (!Tools.isEmpty(msg.get("user_id")))
res = msg.getString("user_id");
return res;
}
int size = wechatSendService.sendTemplateByProject(projectName, templateData);
if (size == -1) {
// result.put("message", "发送微信预警失败，用户获取问题");
// result.put("status", false);
System.out.println("发送微信预警失败，用户获取问题");
} else if (size > 0) {
// result.put("message", "本次微信预警有"+size+"条发送失败");
// result.put("status", false);
System.out.println("本次微信预警有" + size + "条发送失败");
} else {
// result.put("message", "发送微信预警成功");
// result.put("status", true);
System.out.println("发送微信预警成功");
trackRule.setWarn(false);
trackRuleDao.updateTrackrule(trackRule.get_id());
/**
* 获取追踪关键词比较的文本
*
* @Title: getMsgAllText
* @Description: 获取追踪关键词比较的文本
* @param @param
* msg
* @param @return
* 设定文件
* @return String 返回类型
*/
private String getMsgAllText(JSONObject msg) {
String res = "";
StringBuilder buffer = new StringBuilder("");
if (!Tools.isEmpty(msg.get("markPt"))) {
switch (msg.getString("markPt")) {
case "微博":
if (!Tools.isEmpty(msg.get("text")))
buffer.append(msg.getString("text"));
if (!Tools.isEmpty(msg.get("roottext")))
buffer.append(msg.getString("roottext"));
break;
case "知乎":
if (!Tools.isEmpty(msg.get("question_title")))
buffer.append(msg.getString("question_title"));
if (!Tools.isEmpty(msg.get("answer_content")))
buffer.append(msg.getString("answer_content"));
break;
case "视频":
if (!Tools.isEmpty(msg.get("title")))
buffer.append(msg.getString("title"));
if (!Tools.isEmpty(msg.get("content")))
buffer.append(msg.getString("content"));
break;
case "网媒":
if (!Tools.isEmpty(msg.get("title")))
buffer.append(msg.getString("title"));
if (!Tools.isEmpty(msg.get("content")))
buffer.append(msg.getString("content"));
break;
default:
break;
}
res = buffer.toString();
res = Tools.replaceHighLight(res);
}
return res;
}
@Override
public SearchHits keyWordsTrackHitNew(TrackRule trackRule, int count, long startid, Long endid,
PlatformNew platform) {
SearchHits searchHits = null;
String anyWord = "";
String allWords = "";
// 取关键词
if (trackRule.getAndOr().equals("或")) {
anyWord = trackRule.getKeyWordsInputOne() + "，" + trackRule.getKeyWordsInputTwo();
} else if (trackRule.getAndOr().equals("且")) {
allWords = trackRule.getKeyWordsInputOne() + "，" + trackRule.getKeyWordsInputTwo();
} else {
allWords = trackRule.getKeyWordsInputOne();
}
// 取高亮颜色
String color = "#ff330D";
if (null != trackRule.getHighlighted() && "" != trackRule.getHighlighted()) {
color = trackRule.getHighlighted();
}
// 根据平台获取数据
searchHits = trackESDao.getkeyWordsTrackFromEs(anyWord, allWords, color, count, startid, endid, platform);
// if (platformName.equals("微博")) {
// searchHits = trackESDao.getkeyWordsTrackWeiboFromEs(anyWord, allWords, color,
// count, startid, endid);
// } else if (platformName.equals("知乎")) {
// searchHits = trackESDao.getkeyWordsTrackZhihuFromEs(anyWord, allWords, color,
// count, startid, endid);
// } else if (platformName.equals("视频")) {
// searchHits = trackESDao.getkeyWordsTrackVideoFromEs(anyWord, allWords, color,
// count, startid, endid);
// } else {
// searchHits = trackESDao.getkeyWordsTrackMediaFromEs(anyWord, allWords, color,
// count, startid, endid,
// platformName);
// }
return searchHits;
}
@Override
public SearchHits channelTrackHitNew(TrackRule trackRule, int count, long startid, Long endid,
PlatformNew platform) {
SearchHits searchHits = null;
// 取渠道平台
String pt = trackRule.getChannelPt();
String channel = trackRule.getChannelQd();
// 取高亮颜色
String color = "#ff330D";
if (null != trackRule.getHighlighted() && "" != trackRule.getHighlighted()) {
color = trackRule.getHighlighted();
}
String platformName = platform.getPlatformName();
if (pt.equals(platformName)) {
searchHits = trackESDao.getchannelTrackFromEs(channel, color, count, startid, endid, platform);
}
// if (pt.equals("微博")) {
// searchHits = trackESDao.getchannelTrackWeiboFromEs(channel, color, count,
// startid, endid);
// } else if (pt.equals("知乎")) {
// searchHits = trackESDao.getchannelTrackZhihuFromEs(channel, color, count,
// startid, endid);
// } else if (pt.equals("视频")) {
// searchHits = trackESDao.getchannelTrackVideoFromEs(channel, color, count,
// startid, endid);
// } else {
// searchHits = trackESDao.getchannelTrackMediaFromEs(channel, color, count,
// startid, endid, platformName);
// }
return searchHits;
}
@Override
public SearchHits articleTrackHitNew(TrackRule trackRule, int count, long startid, Long endid,
PlatformNew platform) {
SearchHits searchHits = null;
// 取相似新闻数信息
String articleTitle = trackRule.getArticleTitle();
String timeRange = trackRule.getTimeRange();
long createAt = trackRule.getCreateAt();
// 算时间区段(按commonid
if (timeRange.equals("3小时内")) {
// startid = createAt;
// endid = createAt + 3 * 60 * 60 * 1000L;
startid = esGetCommonId.getCommonId(3 * 60 * 60 * 1000L);
} else if (timeRange.equals("6小时内")) {
// startid = createAt;
// endid = createAt + 6 * 60 * 60 * 1000L;
startid = esGetCommonId.getCommonId(6 * 60 * 60 * 1000L);
} else if (timeRange.equals("12小时内")) {
// startid = createAt;
// endid = createAt + 12 * 60 * 60 * 1000L;
startid = esGetCommonId.getCommonId(12 * 60 * 60 * 1000L);
} else if (timeRange.equals("24小时内")) {
// startid = createAt;
// endid = createAt + 24 * 60 * 60 * 1000L;
startid = esGetCommonId.getCommonId(24 * 60 * 60 * 1000L);
}
// 取高亮颜色
String color = "#ff330D";
if (null != trackRule.getHighlighted() && "" != trackRule.getHighlighted()) {
color = trackRule.getHighlighted();
/**
* 关键词判断预警
*
* @Title: isKeywordWarn
* @Description: 关键词判断预警
* @param @param
* content
* @param @param
* keyword
* @param @return
* 设定文件
* @return boolean 返回类型
*/
private boolean isKeywordWarn(String content, String keyword) {
return Tools.eventIsHitkeyword(content, keyword);
}
searchHits = trackESDao.getarticleTrackFromEs(articleTitle, color, count, startid, endid, platform);
// if (platformName.equals("微博")) {
// searchHits = trackESDao.getarticleTrackWeiboFromEs(articleTitle, color,
// count, startid, endid);
// } else if (platformName.equals("知乎")) {
// searchHits = null;
// } else if (platformName.equals("视频")) {
// searchHits = trackESDao.getarticleTrackVideoFromEs(articleTitle, color,
// count, startid, endid);
// ;
// } else {
// searchHits = trackESDao.getarticleTrackMediaFromEs(articleTitle, color,
// count, startid, endid,
// platformName);
// }
return searchHits;
/**
* 渠道判断预警
*
* @Title: isChannelWarn
* @Description: 渠道判断预警
* @param @param
* channelPt
* @param @param
* channelName
* @param @param
* channelId
* @param @param
* channel
* @param @return
* 设定文件
* @return boolean 返回类型
*/
private boolean isChannelWarn(String channelPt, String channelName, String channelId,
List<Map<String, Object>> channel) {
boolean res = false;
for (int i = 0; i < channel.size(); i++) {
Map<String, Object> clInfo = channel.get(i);
if (channelPt.equals(clInfo.get("channelPt")) || "全部".equals(clInfo.get("channelPt"))) {
boolean isNameHit = false;
boolean isIdHit = false;
if (!Tools.isEmpty(clInfo.get("channelName")) && !Tools.isEmpty(clInfo.get("channelId"))) {
// 双重条件
isNameHit = Objects.equals(channelName, clInfo.get("channelName"));
isIdHit = Objects.equals(channelId, clInfo.get("channelId"));
res = (isNameHit && isIdHit);
} else if (Tools.isEmpty(clInfo.get("channelName")) && !Tools.isEmpty(clInfo.get("channelId"))) {
// 单特征id
res = Objects.equals(channelId, clInfo.get("channelId"));
} else if (!Tools.isEmpty(clInfo.get("channelName")) && Tools.isEmpty(clInfo.get("channelId"))) {
// 单渠道名
res = Objects.equals(channelName, clInfo.get("channelName"));
}
if (res)
return res;
}
}
return res;
}
@Override
public void WarnWechatNew(SearchHits trackHit, String platformName, TrackRule trackRule, String projectName) {
if (null != trackHit) {
for (SearchHit hit : trackHit) {
String url = "";
String title = "";
String content = "";
String source = "";
String time = "";
String pt = "";
int fensi = -1;
String type = hit.getType();
Long rstime = null != hit.getSource().get("rstime") ? Long.valueOf(hit.getSource().get("rstime") + "")
: null;
if (type.equals("status") && null != rstime) {
// 微博
url = hit.getSource().get("url") + "";
title = "";
content = hit.getSource().get("text") + "";
source = hit.getSource().get("username") + "";
time = hit.getSource().get("time") + "";
pt = "微博";
fensi = Integer.valueOf(hit.getSource().get("fensi") + "");
} else if (type.equals("zhihu")) {
// zhihu
url = hit.getId() + "";
title = hit.getSource().get("question_title") + "";
content = hit.getSource().get("answer_content") + "";
source = hit.getSource().get("user_name") + "";
time = hit.getSource().get("created_at") + "";
pt = "知乎";
} else if (type.equals("video")) {
// 视频
url = hit.getId() + "";
title = hit.getSource().get("title") + "";
content = "";
source = hit.getSource().get("uper") + "";
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String timein = String.valueOf(hit.getSource().get("time"));
String timeout = timein.replaceAll("T", " ");
try {
Date date = sdf.parse(timeout);
try {
// "2017-09-28 20:35:33"
Date mediadate = TimeUtil.parseTime(TimeUtil.formatTime(TimeUtil.addTimeHour(date, 8)),
"yyyy-MM-dd HH:mm:ss");
time = sdf.format(mediadate);
} catch (Exception e) {
time = sdf.format(System.currentTimeMillis());
}
} catch (ParseException e) {
e.printStackTrace();
}
pt = "视频";
public void warnMsg(JSONObject msg, TrackRule tr) {
Update update = new Update();
if ("wechat".equals(tr.getWarnRule())) {
// 微信预警判断limit是否超过 是否使用
tr = trackRuleDao.getTrackRuleById(tr.get_id());
if ((tr.getWarnLimit() <= tr.getWarnNum()) || (!tr.getIsUsed())) {
// 当前时间超过预警结束时间，自动结束
tr.setIsUsed(false);
tr.setUpdateAt(System.currentTimeMillis());
tr.setWarnNum(0);
endTrackrule(tr);
} else {
// 网媒
String trueid = hit.getSource().get("id") != null ? String.valueOf(hit.getSource().get("id"))
: String.valueOf(hit.getId());
url = trueid;
title = hit.getSource().get("title") + "";
content = hit.getSource().get("content") + "";
source = hit.getSource().get("source") + "";
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String timein = String.valueOf(hit.getSource().get("time"));
String timeout = timein.replaceAll("T", " ");
try {
Date date = sdf.parse(timeout);
try {
// "2017-09-28 20:35:33"
Date mediadate = TimeUtil.parseTime(TimeUtil.formatTime(TimeUtil.addTimeHour(date, 8)),
"yyyy-MM-dd HH:mm:ss");
time = sdf.format(mediadate);
} catch (Exception e) {
time = sdf.format(System.currentTimeMillis());
// 按规则id添加到缓存中（已追踪）
redisService.addWarnMsg2Set(tr.get_id(), msg, TRACKMAX);
update.inc("hasTrack", 1);
warnWechat(msg, tr);
update.inc("warnNum", 1);
Query query = new Query(Criteria.where("_id").is(tr.get_id()));
trackRuleDao.updateTrackrule(query, update);
}
} catch (ParseException e) {
e.printStackTrace();
} else {
// 按规则id添加到缓存中（已追踪）
redisService.addWarnMsg2Set(tr.get_id(), msg, TRACKMAX);
update.inc("hasTrack", 1);
Query query = new Query(Criteria.where("_id").is(tr.get_id()));
trackRuleDao.updateTrackrule(query, update);
}
pt = platformName;
}
TemplateData templateData = new TemplateData();
if (trackRule.getRuleType().equals("keyWords")) {
private void warnWechat(JSONObject msg, TrackRule trackRule) {
// 组装预警预警
String ruleExplain = getTrackRuleExplain(trackRule);
TemplateData templateData = handleTemplateData(msg);
if ("keyword".equals(trackRule.getRuleType())) {
// 关键词追踪
templateData.setWordRule(trackRule.getRuleExplain());
} else if (trackRule.getRuleType().equals("channel")) {
templateData.setWordRule(ruleExplain);
} else if ("channel".equals(trackRule.getRuleType())) {
// 渠道追踪
templateData.setChannelRule(trackRule.getRuleExplain());
}
templateData.setUrl(url);
templateData.setTitle(title);
templateData.setContent(content);
templateData.setSource(source);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");
Date datetime;
try {
datetime = sdf.parse(time);
templateData.setTime(datetime);
} catch (ParseException e) {
e.printStackTrace();
templateData.setTime(new Date());
}
templateData.setPt(pt);
if (fensi != -1) {
templateData.setFensi(fensi);
templateData.setChannelRule(ruleExplain);
}
Map<String, Object> result = new HashMap<String, Object>();
result = userMailService.sendMailByProject(projectName, templateData);
if ((result.get("message") + "").equals("用户组邮件发送成功")) {
log.info("发送邮件预警成功");
trackRule.setWarn(true);
trackRuleDao.updateTrackrule(trackRule.get_id());
break;
} else if ((result.get("message") + "").equals("用户组邮箱未获取到")) {
log.error("发送邮件预警失败，用户获取问题");
for (int i = 0; i < 5; i++) {
int size = wechatSendService.sendTemplateByProject(trackRule.getProject(), templateData);
if (size == -1) {
Tools.sleep(3000L);
log.error("发送微信预警失败，用户获取问题");
} else if (size > 0) {
Tools.sleep(3000L);
log.error("本次微信预警有{}条发送失败", size);
} else {
log.error("本次部分条发送失败");
trackRule.setWarn(true);
trackRuleDao.updateTrackrule(trackRule.get_id());
log.info("发送微信预警成功");
break;
}
}
}
private String getTrackRuleExplain(TrackRule trackRule) {
String ruleContent = "";
if ("keyword".equals(trackRule.getRuleType())) {
// 关键词追踪
ruleContent = trackRule.getKeyword();
} else if ("channel".equals(trackRule.getRuleType())) {
// 渠道追踪
StringBuilder content = new StringBuilder();
List<Map<String, Object>> channels = trackRule.getChannel();
channels.forEach(channel -> {
content.append(channel.get("channelPt"));
content.append("【");
content.append(channel.get("channelName"));
content.append(channel.get("channelId"));
content.append("】,");
});
ruleContent = content.toString();
}
return "规则名：" + trackRule.getRuleName() + "；规则内容：" + ruleContent;
}
@Override
public void WarnEmailNew(SearchHits trackHit, String platformName, TrackRule trackRule, String projectName) {
if (null != trackHit) {
for (SearchHit hit : trackHit) {
private TemplateData handleTemplateData(JSONObject msg) {
String url = "";
String title = "";
String content = "";
......@@ -634,38 +340,55 @@ public class TrackHitAndWarnServiceImpl implements TrackHitAndWarnService {
String time = "";
String pt = "";
int fensi = -1;
String type = hit.getType();
Long rstime = null != hit.getSource().get("rstime") ? Long.valueOf(hit.getSource().get("rstime") + "")
: null;
if (type.equals("status") && null != rstime) {
// 微博
url = hit.getSource().get("url") + "";
if (!Tools.isEmpty(msg.get("markPt"))) {
switch (msg.getString("markPt")) {
case "微博":
if (!Tools.isEmpty(msg.get("url")))
url = msg.getString("url");
title = "";
content = hit.getSource().get("text") + "";
source = hit.getSource().get("username") + "";
time = hit.getSource().get("time") + "";
if (!Tools.isEmpty(msg.get("text")))
content = msg.getString("text");
if (!Tools.isEmpty(msg.get("roottext")))
content = content + "\n原发内容：" + msg.getString("roottext");
if (!Tools.isEmpty(msg.get("username")))
source = msg.getString("username");
if (!Tools.isEmpty(msg.get("time")))
time = msg.getString("time");
pt = "微博";
fensi = Integer.valueOf(hit.getSource().get("fensi") + "");
} else if (type.equals("zhihu")) {
// zhihu
url = hit.getId() + "";
title = hit.getSource().get("question_title") + "";
content = hit.getSource().get("answer_content") + "";
source = hit.getSource().get("user_name") + "";
time = hit.getSource().get("created_at") + "";
if (!Tools.isEmpty(msg.get("fensi")))
fensi = Integer.parseInt(msg.getString("fensi"));
break;
case "知乎":
if (!Tools.isEmpty(msg.get("id")))
url = msg.getString("id");
if (!Tools.isEmpty(msg.get("question_title")))
title = msg.getString("question_title");
if (!Tools.isEmpty(msg.get("answer_content")))
content = msg.getString("answer_content");
if (!Tools.isEmpty(msg.get("user_name")))
source = msg.getString("user_name");
if (!Tools.isEmpty(msg.get("created_at")))
time = msg.getString("created_at");
pt = "知乎";
} else if (type.equals("video")) {
// 视频
url = hit.getId() + "";
title = hit.getSource().get("title") + "";
content = "";
source = hit.getSource().get("uper") + "";
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String timein = String.valueOf(hit.getSource().get("time"));
String timeout = timein.replaceAll("T", " ");
break;
case "视频":
if (!Tools.isEmpty(msg.get("id")))
url = msg.getString("id");
if (!Tools.isEmpty(msg.get("title")))
title = msg.getString("title");
if (!Tools.isEmpty(msg.get("content")))
content = msg.getString("content");
if (!Tools.isEmpty(msg.get("pt")) && "视频".equals(msg.getString("pt"))) {
if (!Tools.isEmpty(msg.get("uper")))
source = msg.getString("uper");
} else {
if (!Tools.isEmpty(msg.get("source")))
source = msg.getString("source");
}
if (!Tools.isEmpty(msg.get("time"))) {
String timein = msg.getString("time");
try {
Date date = sdf.parse(timeout);
Date date = sdfUS.parse(timein);
try {
// "2017-09-28 20:35:33"
Date mediadate = TimeUtil.parseTime(TimeUtil.formatTime(TimeUtil.addTimeHour(date, 8)),
......@@ -676,21 +399,25 @@ public class TrackHitAndWarnServiceImpl implements TrackHitAndWarnService {
}
} catch (ParseException e) {
e.printStackTrace();
time = sdf.format(System.currentTimeMillis());
}
pt = "视频";
} else {
}
pt = msg.getString("pt");
break;
default:
// 网媒
String trueid = hit.getSource().get("id") != null ? String.valueOf(hit.getSource().get("id"))
: String.valueOf(hit.getId());
url = trueid;
title = hit.getSource().get("title") + "";
content = hit.getSource().get("content") + "";
source = hit.getSource().get("source") + "";
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String timein = String.valueOf(hit.getSource().get("time"));
String timeout = timein.replaceAll("T", " ");
if (!Tools.isEmpty(msg.get("id")))
url = msg.getString("id");
if (!Tools.isEmpty(msg.get("title")))
title = msg.getString("title");
if (!Tools.isEmpty(msg.get("content")))
content = msg.getString("content");
if (!Tools.isEmpty(msg.get("source")))
source = msg.getString("source");
if (!Tools.isEmpty(msg.get("time"))) {
String timein = msg.getString("time");
try {
Date date = sdf.parse(timeout);
Date date = sdfUS.parse(timein);
try {
// "2017-09-28 20:35:33"
Date mediadate = TimeUtil.parseTime(TimeUtil.formatTime(TimeUtil.addTimeHour(date, 8)),
......@@ -701,52 +428,29 @@ public class TrackHitAndWarnServiceImpl implements TrackHitAndWarnService {
}
} catch (ParseException e) {
e.printStackTrace();
time = sdf.format(System.currentTimeMillis());
}
pt = platformName;
}
TemplateData templateData = new TemplateData();
if (trackRule.getRuleType().equals("keyWords")) {
// 关键词追踪
templateData.setWordRule(trackRule.getRuleExplain());
} else if (trackRule.getRuleType().equals("channel")) {
// 渠道追踪
templateData.setChannelRule(trackRule.getRuleExplain());
}
templateData.setUrl(url);
templateData.setTitle(title);
templateData.setContent(content);
templateData.setSource(source);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");
Date datetime;
if (!Tools.isEmpty(msg.get("type")))
pt = msg.getString("type");
break;
}
}
Date datetime = new Date();
try {
datetime = sdf.parse(time);
templateData.setTime(datetime);
} catch (ParseException e) {
e.printStackTrace();
templateData.setTime(new Date());
}
templateData.setPt(pt);
datetime = new Date();
}
// content title 处理
content = Tools.replaceHighLight(content);
if (content.length() > 50)
content = content.substring(0, 50);
title = Tools.replaceHighLight(title);
TemplateData res = new TemplateData(url, title, content, source, datetime, pt);
if (fensi != -1) {
templateData.setFensi(fensi);
}
Map<String, Object> result = new HashMap<String, Object>();
result = userMailService.sendMailByProject(projectName, templateData);
if ((result.get("message") + "").equals("用户组邮件发送成功")) {
log.info("发送邮件预警成功");
trackRule.setWarn(true);
trackRuleDao.updateTrackrule(trackRule.get_id());
break;
} else if ((result.get("message") + "").equals("用户组邮箱未获取到")) {
log.error("发送邮件预警失败，用户获取问题");
} else {
log.error("本次部分条发送失败");
trackRule.setWarn(true);
trackRuleDao.updateTrackrule(trackRule.get_id());
break;
}
}
res.setFensi(fensi);
}
return res;
}
}
src/main/java/com/zhiwei/messageflow/mongo/bean/TrackRule.java
package com.zhiwei.messageflow.mongo.bean;
import java.util.List;
import java.util.Map;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
......@@ -10,25 +13,74 @@ import lombok.ToString;
@ToString
@Document(collection = "qbjc_trackrule")
public class TrackRule {
/**
* id用户id
*/
@Id
private long _id;
private String _id;
/**
* ruleName 规则名
*/
private String ruleName;
private String ruleExplain;
/**
* ruleType 规则类型
*/
private String ruleType;
private String KeyWordsInputOne;
private String andOr;
private String KeyWordsInputTwo;
private String earlyWarning;
/**
* keyword 关键词：xxx xx，xx xxx
*/
private String keyword;
/**
* channel List<Map> 规则内容 渠道：平台 渠道名 主键
*/
private List<Map<String, Object>> channel;
/**
* warnRule 预警规则 （normal wechat）
*/
private String warnRule;
/**
* startTime 预警起始时间
*/
private Long startTime;
/**
* endTime long 预警结束时间
*/
private Long endTime;
/**
* highlighted String 高亮颜色
*/
private String highlighted;
private String earlyWarningTime;
private boolean isWarn;
private long createAt;
/**
* createAt long 创建时间
*/
private Long createAt;
/**
* updateAt long 更新时间
*/
private Long updateAt;
/**
* submitter String 提交人
*/
private String submitter;
/**
* isUsed boolean 是否使用
*/
private Boolean isUsed;
/**
* project String 所属项目
*/
private String project;
private String channelPt;
private String channelQd;
private String articleTitle;
private String articleNum;
private String timeRange;
/**
* warnLimit int 预警上限
*/
private Integer warnLimit;
/**
* WarnNum int 本次预警条数
*/
private Integer warnNum;
/**
* hasTrackNum int 已追踪总条数
*/
private Integer hasTrack;
}
src/main/java/com/zhiwei/messageflow/mongo/bean/WechatCode.java
......@@ -10,7 +10,7 @@ import lombok.ToString;
@Data
@ToString
@Document(collection = "wechatCode")
@Document(collection = "wechatCodeNew")
public class WechatCode {
@Id
private String _id;
......
src/main/java/com/zhiwei/messageflow/mongo/dao/TrackRuleDao.java
......@@ -2,6 +2,9 @@ package com.zhiwei.messageflow.mongo.dao;
import java.util.List;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import com.zhiwei.messageflow.mongo.bean.TrackRule;
public interface TrackRuleDao {
......@@ -15,9 +18,22 @@ public interface TrackRuleDao {
List<TrackRule> getTrackRuleByProject(String projectName);
/**
* 更新项目预警情况
*
* @param trackRule_id
* 更新单条追踪规则按条件和更新条件
* @Title: updateTrackrule
* @Description: 更新单条追踪规则
* @param @param query
* @param @param update
* @param @return 设定文件
* @return boolean 返回类型
*/
boolean updateTrackrule(Query query, Update update);
/**
* 按id去追踪规则
* @Title: getTrackRuleById
* @Description: 按id去追踪规则
* @param @param get_id
* @param @return 设定文件
* @return TrackRule 返回类型
*/
void updateTrackrule(Long trackRule_id);
TrackRule getTrackRuleById(String id);
}
src/main/java/com/zhiwei/messageflow/mongo/dao/impl/TrackRuleDaoImpl.java
......@@ -22,18 +22,25 @@ public class TrackRuleDaoImpl implements TrackRuleDao {
@Override
public List<TrackRule> getTrackRuleByProject(String projectName) {
Query query = new Query();
query.addCriteria(Criteria.where("project").is(projectName));
primaryMongoTemplate.getCollectionName(TrackRule.class);
List<TrackRule> trackRules = primaryMongoTemplate.find(query, TrackRule.class);
return trackRules;
Query query = new Query(Criteria.where("project").is(projectName));
return primaryMongoTemplate.find(query, TrackRule.class);
}
@Override
public void updateTrackrule(Long trackRule_id) {
// System.out.println("预警状态更新");
primaryMongoTemplate.updateFirst(Query.query(Criteria.where("_id").is(trackRule_id)),
new Update().set("isWarn", true), TrackRule.class);
public boolean updateTrackrule(Query query, Update update) {
TrackRule t = primaryMongoTemplate.findOne(query, TrackRule.class);
if (t != null) {
primaryMongoTemplate.updateFirst(query, update, TrackRule.class);
} else {
return false;
}
return true;
}
@Override
public TrackRule getTrackRuleById(String id) {
Query query = new Query(Criteria.where("_id").is(id));
return primaryMongoTemplate.findOne(query, TrackRule.class);
}
}
src/main/java/com/zhiwei/messageflow/redis/RedisPoolAndTools.java
......@@ -398,4 +398,31 @@ public class RedisPoolAndTools {
return res;
}
public Set<String> zrevrange(String redisKey, int start, int end) {
Jedis jedis = getJedis();
while (true) {
if (null != jedis) {
break;
} else {
jedis = getJedis();
}
}
Set<String> res = jedis.zrevrange(redisKey, start, end);
returnResource(jedis);
return res;
}
public void zremrangebyscore(String key, long start, long end) {
Jedis jedis = getJedis();
while (true) {
if (null != jedis) {
break;
} else {
jedis = getJedis();
}
}
jedis.zremrangeByScore(key, start, end);
returnResource(jedis);
}
}
\ No newline at end of file
src/main/java/com/zhiwei/messageflow/redis/service/RedisService.java
......@@ -189,5 +189,14 @@ public interface RedisService {
* @return boolean 返回类型
*/
boolean dropCollectionHash(String id);
/**
* 按追踪id添加已追踪消息
* @Title: addWarnMsg2Set
* @Description: 按追踪id添加已追踪消息
* @param @param id
* @param @param msg 设定文件
* @return void 返回类型
*/
boolean addWarnMsg2Set(String id, JSONObject msg, int maxSize);
}
src/main/java/com/zhiwei/messageflow/redis/service/impl/RedisServiceImpl.java
package com.zhiwei.messageflow.redis.service.impl;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
......@@ -11,6 +12,7 @@ import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.zhiwei.messageflow.ESGetCommonId;
import com.zhiwei.messageflow.bean.MediaMessage;
import com.zhiwei.messageflow.bean.VideoMessage;
import com.zhiwei.messageflow.bean.WeiboMessage;
......@@ -151,27 +153,45 @@ public class RedisServiceImpl implements RedisService {
/**
* 删除超出存储上限的数据
*/
removeOverLimit(redisKey, maxSize);
}
private void removeOverLimit(String redisKey, int maxSize) {
long nowCount = redisPoolAndTools.getNowCount(redisKey);// 当前数据量
int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量
if (maxSize > 0 && nowCount > maxSize) {
// 判断zrevrange取maxSize条的commonid是否大于limitCommonid（小于或等于为以maxSize作为limit，大于以limitCommonid作为limit）
try {
Set<String> maxSizeSet = redisPoolAndTools.zrevrange(redisKey, maxSize - 1, maxSize - 1);
JSONObject object = JSONObject.parseObject((new ArrayList<>(maxSizeSet)).get(0));
long commonid = object.getLongValue("commonid");
if (commonid > ESGetCommonId.START_COMMONID) {
redisPoolAndTools.zremrangebyscore(redisKey, 0L,ESGetCommonId.START_COMMONID);
} else {
redisPoolAndTools.removeDataByName(redisKey, removeIndex);
}
} catch (Exception e) {
e.printStackTrace();
redisPoolAndTools.removeDataByName(redisKey, removeIndex);
}
}
}
@Override
public String getDirectRedisKey(String projectName, String groupName, String pt) {
return RedisConfig.DIRECTKEY+projectName+"-"+groupName+"-"+pt;
return RedisConfig.DIRECTKEY + projectName + "-" + groupName + "-" + pt;
}
@Override
public String getDirectRsidMapKey(String projectName) {
return RedisConfig.DIRECTKEY+projectName;
return RedisConfig.DIRECTKEY + projectName;
}
@Override
public boolean insertEvent(JSONObject ob) {
String redisKey = RedisConfig.EVENTKEY+ob.getString("eventId").replace(":", "：");
String redisKey = RedisConfig.EVENTKEY + ob.getString("eventId").replace(":", "：");
try {
redisPoolAndTools.sortedSetZadd(redisKey, (double) ob.getLong("offset"), mapper.writeValueAsString(ob));
} catch (JsonProcessingException e) {
......@@ -191,18 +211,18 @@ public class RedisServiceImpl implements RedisService {
@Override
public List<String> getNeedAutoMark() {
String redisKey = RedisConfig.EVENTLISTKEY;
return redisPoolAndTools.Lrange(redisKey,0,9);
return redisPoolAndTools.Lrange(redisKey, 0, 9);
}
@Override
public int countCollectionData(String id) {
String redisKey = RedisConfig.EVENTKEY+id.replace(":", "：");
String redisKey = RedisConfig.EVENTKEY + id.replace(":", "：");
return Integer.parseInt(String.valueOf(redisPoolAndTools.getNowCount(redisKey)));
}
@Override
public Set<String> getCollectionData(String id, int start, int end) {
String redisKey = RedisConfig.EVENTKEY+id.replace(":", "：");
String redisKey = RedisConfig.EVENTKEY + id.replace(":", "：");
return redisPoolAndTools.zrange(redisKey, start, end);
}
......@@ -215,26 +235,41 @@ public class RedisServiceImpl implements RedisService {
@Override
public boolean existsCollectionHashByUrlkey(String url, String id) {
String redisKey = RedisConfig.EVENTHASHKEY+id;
if(redisPoolAndTools.exists(redisKey)) {
//判断是否有hash urlkey
if(redisPoolAndTools.hexists(redisKey,url)) {
String redisKey = RedisConfig.EVENTHASHKEY + id;
if (redisPoolAndTools.exists(redisKey)) {
// 判断是否有hash urlkey
if (redisPoolAndTools.hexists(redisKey, url)) {
return true;
}else {
redisPoolAndTools.hset(redisKey,url);
} else {
redisPoolAndTools.hset(redisKey, url);
return false;
}
}else {
//新建hash
redisPoolAndTools.hset(redisKey,url);
} else {
// 新建hash
redisPoolAndTools.hset(redisKey, url);
return false;
}
}
@Override
public boolean dropCollectionHash(String id) {
String redisKey = RedisConfig.EVENTHASHKEY+id;
String redisKey = RedisConfig.EVENTHASHKEY + id;
return redisPoolAndTools.del(redisKey);
}
@Override
public boolean addWarnMsg2Set(String id, JSONObject jo, int maxSize) {
String redisKey = RedisConfig.TRACKKEY + id;
redisPoolAndTools.sortedSetZadd(redisKey, (double) jo.getLongValue("commonid"), jo.toJSONString());
/**
* 删除超出存储上限的数据
*/
long nowCount = redisPoolAndTools.getNowCount(redisKey);// 当前数据量
int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量
if (maxSize > 0 && nowCount > maxSize) {
redisPoolAndTools.removeDataByName(redisKey, removeIndex);
}
return true;
}
}
src/main/java/com/zhiwei/messageflow/util/Tools.java 0 → 100644
package com.zhiwei.messageflow.util;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.math.BigDecimal;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.security.MessageDigest;
import java.security.SecureRandom;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.crypto.Cipher;
import javax.crypto.KeyGenerator;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class Tools {
private static final Logger log = LoggerFactory.getLogger(Tools.class);
private static final String key = "3e5135ac50dabd5476b5d1107fa839f9";
private static final long timeout = 3 * 60 * 1000;
private static final char DBC_CHAR_START = 33; // 半角!
private static final char DBC_CHAR_END = 126; // 半角~
private static final char SBC_CHAR_START = 65281; // 全角！
private static final char SBC_CHAR_END = 65374; // 全角～
private static final int CONVERT_STEP = 65248; // 全角半角转换间隔
private static final char SBC_SPACE = 12288; // 全角空格 12288
private static final char DBC_SPACE = ' '; // 半角空格
public static String sendGet(String url, String param) {
String result = "";
BufferedReader in = null;
try {
String urlName = url + "?" + param;
URL realUrl = new URL(urlName);
// 打开和URL之间的连接
URLConnection conn = realUrl.openConnection();
// 设置通用的请求属性
conn.setRequestProperty("accept", "*/*");
conn.setRequestProperty("connection", "Keep-Alive");
conn.setRequestProperty("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)");
// 建立实际的连接
conn.connect();
// 获取所有响应头字段
Map<String, List<String>> map = conn.getHeaderFields();
// 遍历所有的响应头字段
// for (String key : map.keySet()) {
// System.out.println(key + "--->" + map.get(key));
// }
// 定义BufferedReader输入流来读取URL的响应
in = new BufferedReader(new InputStreamReader(conn.getInputStream()));
String line;
while ((line = in.readLine()) != null) {
result += line;
}
} catch (Exception e) {
log.error("发送GET请求出现异常！", e);
e.printStackTrace();
}
// 使用finally块来关闭输入流
finally {
try {
if (in != null) {
in.close();
}
} catch (IOException ex) {
ex.printStackTrace();
}
}
return result;
}
/**
* <PRE>
*
* 半角字符->全角字符转换
* 只处理空格，!到˜之间的字符，忽略其他
* </PRE>
*/
public static String DBC2SBC(String src) {
if (src == null) {
return src;
}
StringBuilder buf = new StringBuilder(src.length());
char[] ca = src.toCharArray();
for (int i = 0; i < ca.length; i++) {
if (ca[i] == DBC_SPACE) { // 如果是半角空格，直接用全角空格替代
buf.append(SBC_SPACE);
} else if ((ca[i] >= DBC_CHAR_START) && (ca[i] <= DBC_CHAR_END)) { // 字符是!到~之间的可见字符
buf.append((char) (ca[i] + CONVERT_STEP));
} else { // 不对空格以及ascii表中其他可见字符之外的字符做任何处理
buf.append(ca[i]);
}
}
return buf.toString();
}
/**
* <PRE>
*
* 全角字符->半角字符转换
* 只处理全角的空格，全角！到全角～之间的字符，忽略其他
* </PRE>
*/
public static String SBC2DBC(String src) {
if (src == null) {
return src;
}
StringBuilder buf = new StringBuilder(src.length());
char[] ca = src.toCharArray();
for (int i = 0; i < src.length(); i++) {
if (ca[i] >= SBC_CHAR_START && ca[i] <= SBC_CHAR_END) { // 如果位于全角！到全角～区间内
buf.append((char) (ca[i] - CONVERT_STEP));
} else if (ca[i] == SBC_SPACE) { // 如果是全角空格
buf.append(DBC_SPACE);
} else { // 不处理全角空格，全角！到全角～区间外的字符
buf.append(ca[i]);
}
}
return buf.toString();
}
/**
* ccc
*
* @Title: getFileType
* @Description: TODO(判断文件类型)
* @param type
* @param @return
* 设定文件ccc
* @return String 返回类型
*/
public static String getFileType(String path) {
String result = "";
if (path != null) {
int index = 0;
try {
if ((index = path.lastIndexOf('.')) != -1) {
result = path.substring(index + 1);
}
} catch (Exception e) {
result = "";
}
}
result = result.toLowerCase();
return result;
}
/**
* @Title: getMIME
* @Description: TODO(获取MIME类型)
* @param type
* @param @return
* 设定文件
* @return String 返回类型
*/
public static String getMIME(String type) {
String result = null;
type = type.toLowerCase();
if (type.equals("html") || type.equals("htm"))
result = "text/html";
if (type.equals("xml"))
result = "text/xml";
if (type.equals("xhtml"))
result = "application/xhtml+xml";
if (type.equals("txt"))
result = "text/plain";
if (type.equals("rtf"))
result = "application/rtf";
if (type.equals("pdf"))
result = "application/pdf";
if (type.equals("doc") || type.equals("docx"))
result = "application/msword";
if (type.equals("png"))
result = "image/png";
if (type.equals("jpg") || type.equals("jpeg"))
result = "image/jpeg";
if (type.equals("gif"))
result = "image/gif";
return result;
}
/**
* @Title: saveObjectToFile
* @Description: TODO(将对象保存到文件)
* @param path
* @param obj
* @param @return
* 设定文件
* @return void 返回类型
*/
public static void saveObjectToFile(String path, Object obj) {
try {
if (!(obj instanceof Serializable))
throw new IOException(obj + " does not implement interface Serializable");
ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(path));
oos.writeObject(obj);
oos.flush();
oos.close();
} catch (Exception e) {
log.error("should implement Serializable", e);
}
}
/**
* @Title: readObjectInFile
* @Description: TODO(读取文件中存储的对象)
* @param path
* @param @return
* 设定文件
* @return Object 返回类型
*/
public static Object readObjectInFile(String path) {
Object obj = null;
try {
ObjectInputStream ois = new ObjectInputStream(new FileInputStream(path));
obj = ois.readObject();
ois.close();
} catch (Exception e) {
File file = new File(path);
file.delete();
log.error("error in readObjectInFile", e);
}
return obj;
}
/**
* @Title: getEncode
* @Description: TODO(判断字符串编码)
* @param path
* @param @return
* 设定文件
* @return String 返回类型
*/
public static String getEncode(String path) {
String result = null;
BufferedInputStream bis = null;
try {
bis = new BufferedInputStream(new FileInputStream(path));
int p = (bis.read() << 8) + bis.read();
switch (p) {
case 0xefbb:
result = "UTF-8";
break;
case 0xfffe:
result = "UTF-16LE";// Unicode
break;
case 0xfeff:
result = "UTF-16BE";
break;
case 0x5c75:
result = "ANSI|ASCII";
default:
result = "GBK";
}
} catch (Exception e) {
log.error("error in getEncode", e);
} finally {
try {
if (bis != null) {
bis.close();
}
} catch (Exception e) {
e.printStackTrace();
}
}
return result;
}
/**
* @Title: validKey
* @Description: TODO(验证下载校验码)
* @param type
* @param @return
* 设定文件
* @return String 返回类型
*/
public static boolean validKey(String id, String token) {
boolean result = false;
if (id != null && token != null) {
String src = decrypt(token);
if (src != null) {
if (src.indexOf(',') != -1) {
String[] strs = src.split(",");
if (strs.length == 2) {
result = true;
try {
if (!strs[0].equals(id))
result = false;
Date date = new Date(Long.parseLong(strs[1]));
if (new Date().getTime() - date.getTime() > timeout)
result = false;
} catch (Exception e) {
result = false;
e.printStackTrace();
}
}
}
}
}
return result;
}
/**
* @Title: generateKey
* @Description: TODO(生成下载校验码)
* @param type
* @param @return
* 设定文件
* @return String 返回类型
*/
public static String generateKey(String id) {
String result = null;
if (id != null) {
String str = id + "," + new Date().getTime();
result = encrypt(str);
}
return result;
}
private static String encrypt(String str) {
String result = null;
if (str != null && key != null) {
try {
KeyGenerator kgen = KeyGenerator.getInstance("AES");
SecureRandom sr = SecureRandom.getInstance("SHA1PRNG");
sr.setSeed(key.getBytes());
kgen.init(128, sr);
SecretKey secretKey = kgen.generateKey();
byte[] enCodeFormat = secretKey.getEncoded();
SecretKeySpec sks = new SecretKeySpec(enCodeFormat, "AES");
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");// 创建密码器
byte[] byteContent = str.getBytes();
cipher.init(Cipher.ENCRYPT_MODE, sks);// 初始化
result = bytesToHexString(cipher.doFinal(byteContent));
} catch (Exception e) {
}
}
return result;
}
private static String decrypt(String str) {
String result = null;
if (str != null && key != null) {
try {
KeyGenerator kgen = KeyGenerator.getInstance("AES");
SecureRandom sr = SecureRandom.getInstance("SHA1PRNG");
sr.setSeed(key.getBytes());
kgen.init(128, sr);
SecretKey secretKey = kgen.generateKey();
byte[] enCodeFormat = secretKey.getEncoded();
SecretKeySpec sks = new SecretKeySpec(enCodeFormat, "AES");
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");// 创建密码器
cipher.init(Cipher.DECRYPT_MODE, sks);// 初始化
result = new String(cipher.doFinal(hexStringToBytes(str)));
} catch (Exception e) {
}
}
return result;
}
/**
* 将二进制转换成16进制
*
* @param buf
* @return
*/
private static String bytesToHexString(byte bytes[]) {
String result = null;
if (bytes != null) {
if (bytes.length > 0) {
StringBuffer sb = new StringBuffer();
for (int i = 0; i < bytes.length; i++) {
String hex = Integer.toHexString(bytes[i] & 0xFF);
if (hex.length() == 1) {
hex = '0' + hex;
}
sb.append(hex);
}
result = sb.toString().toLowerCase();
}
}
return result;
}
/**
* 将16进制转换为二进制
*
* @param hexStr
* @return
*/
private static byte[] hexStringToBytes(String hexs) {
byte[] result = null;
if (hexs != null) {
if (!hexs.equals("")) {
hexs = hexs.toUpperCase();
result = new byte[hexs.length() / 2];
for (int i = 0; i < hexs.length() / 2; i++) {
int high = Integer.parseInt(hexs.substring(i * 2, i * 2 + 1), 16);
int low = Integer.parseInt(hexs.substring(i * 2 + 1, i * 2 + 2), 16);
result[i] = (byte) (high * 16 + low);
}
}
}
return result;
}
public static String escapeSql(String str) {
String result = null;
if (str != null) {
if (str.equals(""))
str = null;
}
if (str != null) {
result = StringEscapeUtils.escapeSql(str);
}
return result;
}
/**
* @Title: md5
* @Description: 计算字符串Md5
* @param str
* @param @return
* 设定文件
* @return String 返回类型
*/
public static String md5(String str) {
String result = null;
try {
MessageDigest md = MessageDigest.getInstance("MD5");
byte[] bytes = str.getBytes("utf-8");
md.update(bytes);
bytes = md.digest();
result = bytesToHexString(bytes);
} catch (Exception e) {
}
return result;
}
/**
* @Title: sleep
* @Description: 线程等待，忽略异常
* @param millis
* @return void
*/
public static void sleep(long millis) {
try {
Thread.sleep(millis);
} catch (InterruptedException e) {
}
}
/**
*
* @deprecated:生成邮件验证的code
* @param @return
* @return String 返回类型
*/
public static String generateMailCode(int length) {
return RandomStringUtils.randomAlphanumeric(length);
}
/**
*
* @deprecated:生成手机验证的code
* @param length
* @param @return
* @return String 返回类型
*/
public static String generatePhoneCode(int length) {
return RandomStringUtils.randomNumeric(length);
}
/**
* @Title: replaceBlank
* @Description: TODO(去除特殊字符)
* @param str
* @param @return
* 设定文件
* @return String 返回类型
*/
public static String replaceBlank(Object str) {
String dest = "";
if (str != null) {
Pattern p = Pattern.compile("=|,|\"|“|\\s*|\t|\r|\n");
Matcher m = p.matcher(str.toString());
dest = m.replaceAll("");
}
return dest;
}
/**
*
* @Title: decodeUnicode
* @Description: TODO(unicode 转换成 中文)
* @param theString
* @param @return
* 设定文件
* @return String 返回类型
*/
public static String decodeUnicode(String theString) {
char aChar;
int len = theString.length();
StringBuffer outBuffer = new StringBuffer(len);
for (int x = 0; x < len;) {
aChar = theString.charAt(x++);
if (aChar == '\\') {
aChar = theString.charAt(x++);
if (aChar == 'u') {
// Read the xxxx
int value = 0;
for (int i = 0; i < 4; i++) {
aChar = theString.charAt(x++);
switch (aChar) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
value = (value << 4) + aChar - '0';
break;
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
value = (value << 4) + 10 + aChar - 'a';
break;
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
value = (value << 4) + 10 + aChar - 'A';
break;
default:
throw new IllegalArgumentException("Malformed \\uxxxx encoding.");
}
}
outBuffer.append((char) value);
} else {
if (aChar == 't')
aChar = '\t';
else if (aChar == 'r')
aChar = '\r';
else if (aChar == 'n')
aChar = '\n';
else if (aChar == 'f')
aChar = '\f';
outBuffer.append(aChar);
}
} else
outBuffer.append(aChar);
}
return outBuffer.toString();
}
/**
*
* @Title: percentMap
* @Description: TODO(计算百分比，保留两位小数)
* @param map
* @param @return
* 设定文件
* @return Map<String,String> 返回类型
*/
public static Map<String, String> percentMap(Map<String, Integer> map) {
Map<String, String> newMap = new HashMap<String, String>();
Integer total = 0;
for (Entry<String, Integer> e : map.entrySet()) {
total += Integer.valueOf(e.getValue().toString());
}
for (String str : map.keySet()) {
double f = map.get(str).doubleValue() / total * 100;
// 保留两位小数
BigDecimal b = new BigDecimal(f);
double f1 = b.setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue();
newMap.put(str, f1 + "%");
}
return newMap;
}
/**
*
* @TODO( 保留两位小数)
*
* @author 陈炜涛
* @param map
* @return
* @time 2016年11月24日上午10:07:33
* @return Map<String,String>
*/
public static Double percentDouble(double num) {
double f1 = 0.00;
if (0 != num) {
// 保留两位小数
BigDecimal b = new BigDecimal(num);
f1 = b.setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue();
}
return f1;
}
/**
* @Title: getFile
* @Description: TODO(根据文件名和路径获取文件)
* @param path
* @param fileName
* @param @return
* 设定文件
* @return File 返回类型
*/
public static File getFile(String path, String fileName) {
// 判断目录，不存在那么创建目录
path.replace("\\", File.pathSeparator);
File dir = new File(path);
if (!dir.exists()) {
dir.mkdirs();
}
// 判断文件是否存在
File file = new File(path, fileName);
if (!file.exists()) {
try {
file.createNewFile();
} catch (IOException e) {
e.printStackTrace();
}
}
return file;
}
/**
*
* @Description:判断目录中是否存在文件
* @param @return
* @return File 返回类型
*/
public static File isFile(String dir, String filename) {
File dirFile = new File(dir);
if (!dirFile.exists()) {
dirFile.mkdirs();
}
File file = new File(dir, filename);
if (file.exists()) {
return file;
}
return null;
}
/**
* 删除文件
*
* @Title: deleteFile
* @Description:删除文件
* @param @param
* path
* @param @return
* 设定文件
* @return Boolean 返回类型
*/
public static Boolean deleteFile(String path) {
Boolean succeedDelete = true;
File file = new File(path);
if (file.isFile() && file.exists()) {
for (int i = 0; i < 20; i++) {
succeedDelete = file.delete();
if (succeedDelete) {
// LOGGER.info("尝试删除" + file.getPath() + "的次数：" + (i + 1), ".*");
break;
} else {
System.gc();
try {
Thread.sleep(200);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}
return succeedDelete;
}
// 正则表达式
/**
* #([^\\#|.]+)# 话题的表达式
*
* @用户的表达式 @[\u4e00-\u9fa5a-zA-Z0-9_-]{4,30}
*
*/
public static String getString(String text, String expression) {
StringBuffer buffer = new StringBuffer();
Pattern TAG_PATTERN = Pattern.compile(expression);
Matcher m = TAG_PATTERN.matcher(text);
while (m.find()) {
buffer.append(m.group());
buffer.append(" ");
}
if (!" ".equals(buffer.toString())) {
System.out.println("原始内容：" + text + "\t获取的数据：" + buffer.toString());
return buffer.toString();
} else {
System.out.println("原始内容：" + text.substring(0, 10) + "\t没找到@用户：");
}
return "";
}
/**
* @Title: changeVtype
* @Description: TODO(微博认证类型转换)
* @param vtype
* @param @return
* 设定文件
* @return String 返回类型
*/
public static String changeVtype(int vtype) {
String type = "";
switch (vtype) {
case -2:
type = "未知";
break;
case -1:
type = "普通用户";
break;
case 0:
type = "名人";
break;
case 1:
type = "政府";
break;
case 2:
type = "企业";
break;
case 3:
type = "媒体";
break;
case 4:
type = "校园";
break;
case 5:
type = "网站";
break;
case 6:
type = "应用";
break;
case 7:
type = "团体";
break;
case 10:
type = "微博女郎";
break;
case 220:
type = "达人";
break;
case 200:
type = "达人";
break;
}
return type;
}
/**
*
* @Title: getWechatTsn
* @Description: TODO(更具相差多少天匹配微信采集参数)
* @param day
* @param @return
* 设定文件
* @return int 返回类型
*/
public static int getWechatTsn(int day) {
int tsn = 1;
if (day > 1 && day <= 7) {
tsn = 2;
} else if (day > 7 && day <= 30) {
tsn = 3;
} else if (day > 31) {
tsn = 4;
}
return tsn;
}
/**
* @Title: getWorksByFile
* @Description: TODO(通过文件名获取文件中的关键词)
* @param file
* @param @return
* 设定文件
* @return List<String> 返回类型
*/
public static List<String> getWorksByFile(File file) {
List<String> list = new ArrayList<String>();
FileInputStream fi = null;
try {
fi = new FileInputStream(file);
BufferedReader reader = new BufferedReader(new InputStreamReader(fi, "UTF-8"));
String line = null;
while ((line = reader.readLine()) != null) {
list.add(line);
}
reader.close();
fi.close();
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if (fi != null) {
fi.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
return list;
}
/**
*
* @Title: filterSpecialCharacter
* @Description: TODO(过滤特殊字符)
* @param str
* @param @return
* 设定文件
* @return String 返回类型
*/
public static String filterSpecialCharacter(String str) {
try {
String regEx = "【[`~!@#$%^&*()+=|{}':;'//[//].<>/?~！@#%……&*——+|｛｝“”；：‘’，。、·]】";
Pattern p = Pattern.compile(regEx);
Matcher m = p.matcher(str);
return m.replaceAll("");
} catch (Exception ex) {
return str;
}
}
/**
* 模糊匹配
*
* @Title: approximateStringMatching
* @Description: TODO(这里用一句话描述这个方法的作用)
* @param @param
* beMatching
* @param @param
* mathingWord
* @param @return
* 设定文件
* @return boolean 返回类型
*/
public static boolean approximateStringMatching(String beMatching, String mathingWord) {
boolean res = false;
beMatching = beMatching.toLowerCase();
mathingWord = mathingWord.toLowerCase();
if (beMatching.indexOf(mathingWord) != -1) {
res = true;
}
return res;
}
// 判断是否为16进制数
public static boolean isHex(char c) {
if (((c >= '0') && (c <= '9')) || ((c >= 'a') && (c <= 'f')) || ((c >= 'A') && (c <= 'F')))
return true;
else
return false;
}
/**
* decode转普通%
*
* @Title: convertPercent
* @Description: TODO(这里用一句话描述这个方法的作用)
* @param @param
* str
* @param @return
* 设定文件
* @return String 返回类型
*/
public static String convertPercent(String str) {
StringBuilder sb = new StringBuilder(str);
for (int i = 0; i < sb.length(); i++) {
char c = sb.charAt(i);
// 判断是否为转码符号%
if (c == '%') {
if (((i + 1) < sb.length() - 1) && ((i + 2) < sb.length() - 1)) {
char first = sb.charAt(i + 1);
char second = sb.charAt(i + 2);
// 如只是普通的%则转为%25
if (!(isHex(first) && isHex(second)))
sb.insert(i + 1, "25");
} else {// 如只是普通的%则转为%25
sb.insert(i + 1, "25");
}
}
}
return sb.toString();
}
/**
* 过滤掉一些特殊符号 后期根据实际需求不断添加
*
* @param text
*/
public static String filterSpecialChar(String text) {
return text.replaceAll("[\u200B\u200C\u200D\u0009]", "");
}
/**
* 获取主机名
*
* @param url
* @return
*/
public static String getHost(String url) {
try {
return new URL(url).getHost();
} catch (MalformedURLException e) {
e.printStackTrace();
}
return null;
}
/**
* @Title: isIncludeDay
* @Description: 判断截止时间是否在当天
* @param @param
* endtime(yyyy-mm-dd )
* @return @return boolean
*/
public static boolean isIncludeDay(String endtime) {
boolean res = false;
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
long systime = System.currentTimeMillis();
String sysDate = simpleDateFormat.format(systime);
String endDate = endtime.split(" ")[0];
if (endDate.equals(sysDate)) {
res = true;
}
return res;
}
/**
* "xxx xx，xx xxx"内容判断是否命中关键词
*
* @Title: eventIsHitkeyword
* @Description: "xxx xx，xx xxx"内容判断是否命中关键词
* @param @param
* content
* @param @param
* keyword
* @param @return
* 设定文件
* @return boolean 返回类型
*/
public static boolean eventIsHitkeyword(String content, String keyword) {
boolean res = false;
// 处理自定义关键词,分割或
String[] words = keyword.split("\\，");
for (int i = 0; i < words.length; i++) {
String word = words[i];
// 分割且
String[] keys = word.split(" ");
boolean ishit = true;
for (int j = 0; j < keys.length; j++) {
String key = keys[j];
ishit = Tools.approximateStringMatching(content, key);
if (!ishit)
break;
}
// 如果且关系都有命中则，满足条件
if (ishit) {
res = true;
break;
}
}
return res;
}
/**
* 计算进度
*
* @Title: getSchedule
* @Description: 计算进度
* @param @param
* part
* @param @param
* all
* @param @param
* max
* @param @return
* 设定文件
* @return int 返回类型
*/
public static int getSchedule(int part, int all, int max) {
int schedule = (int) (((double) part / (double) all) * max);
return schedule;
}
/**
* 非空判断
*
* @Title: isEmpty
* @Description: 非空判断
* @param @param
* object
* @param @return
* 设定文件
* @return boolean 返回类型
*/
@SuppressWarnings("rawtypes")
public static boolean isEmpty(Object obj) {
if (null == obj) {
return true;
}
if ((obj instanceof List)) {
return CollectionUtils.isEmpty((List) obj);
}
if ((obj instanceof String)) {
return StringUtils.isEmpty((String) obj);
}
return false;
}
/**
* 去除高亮<font style='font-weight:bold;' color='#ff6c60'></font>
* @Title: replaceHighLight
* @Description: 去除高亮<font style='font-weight:bold;' color='#ff6c60'></font>
* @param @param res
* @param @return 设定文件
* @return String 返回类型
*/
public static String replaceHighLight(String res) {
res = res.replaceAll("<font style='font-weight:bold;' color='#ff6c60'>", "");
res = res.replaceAll("</font>", "");
return res;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment