Commit 832c8657 by shentao

2018/10/12 新版采集消息流线上版1.0.0

parent 01a0668c
...@@ -200,6 +200,14 @@ ...@@ -200,6 +200,14 @@
<plugin> <plugin>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId> <artifactId>spring-boot-maven-plugin</artifactId>
<version>2.0.4.RELEASE</version>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin> </plugin>
</plugins> </plugins>
</build> </build>
......
...@@ -41,7 +41,7 @@ public class HighLightFillingServiceImpl implements HighLightFillingService { ...@@ -41,7 +41,7 @@ public class HighLightFillingServiceImpl implements HighLightFillingService {
@Override @Override
public WeiboMessage getWeiboBean(SearchHit searchHit) { public WeiboMessage getWeiboBean(SearchHit searchHit) {
JSONObject wmJson = getBean(searchHit); JSONObject wmJson = getBean(searchHit);
/** /**
...@@ -54,8 +54,8 @@ public class HighLightFillingServiceImpl implements HighLightFillingService { ...@@ -54,8 +54,8 @@ public class HighLightFillingServiceImpl implements HighLightFillingService {
WeiboMessage wm = new WeiboMessage(); WeiboMessage wm = new WeiboMessage();
try { try {
message = mapper.writeValueAsString(sourceHitMap); message = mapper.writeValueAsString(sourceHitMap);
// JSONObject mesJson = JSONObject.parseObject(message); // JSONObject mesJson = JSONObject.parseObject(message);
// System.out.println(searchHit.getType()); // System.out.println(searchHit.getType());
wm = mapper.readValue(message, WeiboMessage.class); wm = mapper.readValue(message, WeiboMessage.class);
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
...@@ -95,23 +95,23 @@ public class HighLightFillingServiceImpl implements HighLightFillingService { ...@@ -95,23 +95,23 @@ public class HighLightFillingServiceImpl implements HighLightFillingService {
HuserInfoWeibo huserInfoWeibo = huserInfoDao.getHuserInfoWeibo(String.valueOf(sourceHitMap.get("user_id"))); HuserInfoWeibo huserInfoWeibo = huserInfoDao.getHuserInfoWeibo(String.valueOf(sourceHitMap.get("user_id")));
wm.setChannelIndex(huserInfoWeibo == null ? 1.0 : huserInfoWeibo.getChannelIndex()); wm.setChannelIndex(huserInfoWeibo == null ? 1.0 : huserInfoWeibo.getChannelIndex());
// System.err.println("json"+wmJson); // System.err.println("json"+wmJson);
// try { // try {
// System.err.println("mapper"+mapper.writeValueAsString(wm)); // System.err.println("mapper"+mapper.writeValueAsString(wm));
// } catch (JsonProcessingException e) { // } catch (JsonProcessingException e) {
// // TODO Auto-generated catch block // // TODO Auto-generated catch block
// e.printStackTrace(); // e.printStackTrace();
// } // }
return wm; return wm;
} }
@Override @Override
public ZhihuMessage getZhihuBean(SearchHit searchHit) { public ZhihuMessage getZhihuBean(SearchHit searchHit) {
JSONObject wmJson = getBean(searchHit); JSONObject wmJson = getBean(searchHit);
Map<String, Object> sourceHitMap = searchHit.getSource(); Map<String, Object> sourceHitMap = searchHit.getSource();
Map<String, HighlightField> highlightFieldsHitMap = searchHit.getHighlightFields(); Map<String, HighlightField> highlightFieldsHitMap = searchHit.getHighlightFields();
...@@ -235,24 +235,23 @@ public class HighLightFillingServiceImpl implements HighLightFillingService { ...@@ -235,24 +235,23 @@ public class HighLightFillingServiceImpl implements HighLightFillingService {
zm.setImg(imgUrl); zm.setImg(imgUrl);
} }
} }
// System.err.println("json"+wmJson);
// System.err.println("json"+wmJson); // try {
// try { // System.err.println("mapper"+mapper.writeValueAsString(zm));
// System.err.println("mapper"+mapper.writeValueAsString(zm)); // } catch (JsonProcessingException e) {
// } catch (JsonProcessingException e) { // // TODO Auto-generated catch block
// // TODO Auto-generated catch block // e.printStackTrace();
// e.printStackTrace(); // }
// }
return zm; return zm;
} }
@Override @Override
public VideoMessage getVideoBean(SearchHit searchHit) { public VideoMessage getVideoBean(SearchHit searchHit) {
JSONObject wmJson = getBean(searchHit); JSONObject wmJson = getBean(searchHit);
Map<String, Object> sourceHitMap = searchHit.getSource(); Map<String, Object> sourceHitMap = searchHit.getSource();
Map<String, HighlightField> highlightFieldsHitMap = searchHit.getHighlightFields(); Map<String, HighlightField> highlightFieldsHitMap = searchHit.getHighlightFields();
...@@ -290,23 +289,23 @@ public class HighLightFillingServiceImpl implements HighLightFillingService { ...@@ -290,23 +289,23 @@ public class HighLightFillingServiceImpl implements HighLightFillingService {
title = vm.getTitle(); title = vm.getTitle();
} }
vm.setTitle(title); vm.setTitle(title);
// System.err.println("json"+wmJson); // System.err.println("json"+wmJson);
// try { // try {
// System.err.println("mapper"+mapper.writeValueAsString(vm)); // System.err.println("mapper"+mapper.writeValueAsString(vm));
// } catch (JsonProcessingException e) { // } catch (JsonProcessingException e) {
// // TODO Auto-generated catch block // // TODO Auto-generated catch block
// e.printStackTrace(); // e.printStackTrace();
// } // }
return vm; return vm;
} }
@Override @Override
public MediaMessage getMediaBean(SearchHit searchHit) { public MediaMessage getMediaBean(SearchHit searchHit) {
JSONObject wmJson = getBean(searchHit); JSONObject wmJson = getBean(searchHit);
Map<String, Object> sourceHitMap = searchHit.getSource(); Map<String, Object> sourceHitMap = searchHit.getSource();
Map<String, HighlightField> highlightFieldsHitMap = searchHit.getHighlightFields(); Map<String, HighlightField> highlightFieldsHitMap = searchHit.getHighlightFields();
...@@ -385,15 +384,15 @@ public class HighLightFillingServiceImpl implements HighLightFillingService { ...@@ -385,15 +384,15 @@ public class HighLightFillingServiceImpl implements HighLightFillingService {
mm.setChannelIndex(1.0); mm.setChannelIndex(1.0);
} }
} }
// System.err.println("json"+wmJson); // System.err.println("json"+wmJson);
// try { // try {
// System.err.println("mapper"+mapper.writeValueAsString(mm)); // System.err.println("mapper"+mapper.writeValueAsString(mm));
// } catch (JsonProcessingException e) { // } catch (JsonProcessingException e) {
// // TODO Auto-generated catch block // // TODO Auto-generated catch block
// e.printStackTrace(); // e.printStackTrace();
// } // }
return mm; return mm;
} }
...@@ -412,10 +411,11 @@ public class HighLightFillingServiceImpl implements HighLightFillingService { ...@@ -412,10 +411,11 @@ public class HighLightFillingServiceImpl implements HighLightFillingService {
* ES数据读取 * ES数据读取
*/ */
Map<String, Object> sourceHitMap = searchHit.getSource(); Map<String, Object> sourceHitMap = searchHit.getSource();
// Map<String, HighlightField> highlightFieldsHitMap = searchHit.getHighlightFields(); // Map<String, HighlightField> highlightFieldsHitMap =
// searchHit.getHighlightFields();
String message; String message;
JSONObject res =new JSONObject(); JSONObject res = new JSONObject();
try { try {
message = mapper.writeValueAsString(sourceHitMap); message = mapper.writeValueAsString(sourceHitMap);
res = JSONObject.parseObject(message); res = JSONObject.parseObject(message);
...@@ -424,32 +424,34 @@ public class HighLightFillingServiceImpl implements HighLightFillingService { ...@@ -424,32 +424,34 @@ public class HighLightFillingServiceImpl implements HighLightFillingService {
e.printStackTrace(); e.printStackTrace();
} }
String type = searchHit.getType(); String type = searchHit.getType();
Long rstime = null!=searchHit.getSource().get("rstime")?Long.valueOf(searchHit.getSource().get("rstime")+""):null; Long rstime = null != searchHit.getSource().get("rstime")
if(type.equals("status")&&null!=rstime) { ? Long.valueOf(searchHit.getSource().get("rstime") + "")
//微博 : null;
res = handleWeiboBean(searchHit,res); if (type.equals("status") && null != rstime) {
}else if(type.equals("zhihu")) { // 微博
//zhihu res = handleWeiboBean(searchHit, res);
res = handleZhihuBean(searchHit,res); } else if (type.equals("zhihu")) {
}else if(type.equals("video")) { // zhihu
//视频 res = handleZhihuBean(searchHit, res);
res = handleVideoBean(searchHit,res); } else if (type.equals("video")) {
}else { // 视频
//网媒 res = handleVideoBean(searchHit, res);
res = handleMediaBean(searchHit,res); } else {
// 网媒
res = handleMediaBean(searchHit, res);
} }
return res; return res;
} }
private JSONObject handleWeiboBean(SearchHit searchHit, JSONObject res) { private JSONObject handleWeiboBean(SearchHit searchHit, JSONObject res) {
/** /**
* ES数据读取 * ES数据读取
*/ */
Map<String, Object> sourceHitMap = searchHit.getSource(); Map<String, Object> sourceHitMap = searchHit.getSource();
Map<String, HighlightField> highlightFieldsHitMap = searchHit.getHighlightFields(); Map<String, HighlightField> highlightFieldsHitMap = searchHit.getHighlightFields();
String text = ""; String text = "";
String roottext = ""; String roottext = "";
Text[] textlist = highlightFieldsHitMap.containsKey("text") ? highlightFieldsHitMap.get("text").getFragments() Text[] textlist = highlightFieldsHitMap.containsKey("text") ? highlightFieldsHitMap.get("text").getFragments()
...@@ -485,7 +487,7 @@ public class HighLightFillingServiceImpl implements HighLightFillingService { ...@@ -485,7 +487,7 @@ public class HighLightFillingServiceImpl implements HighLightFillingService {
HuserInfoWeibo huserInfoWeibo = huserInfoDao.getHuserInfoWeibo(String.valueOf(sourceHitMap.get("user_id"))); HuserInfoWeibo huserInfoWeibo = huserInfoDao.getHuserInfoWeibo(String.valueOf(sourceHitMap.get("user_id")));
res.put("channelIndex", huserInfoWeibo == null ? 1.0 : huserInfoWeibo.getChannelIndex()); res.put("channelIndex", huserInfoWeibo == null ? 1.0 : huserInfoWeibo.getChannelIndex());
return res; return res;
} }
...@@ -495,18 +497,16 @@ public class HighLightFillingServiceImpl implements HighLightFillingService { ...@@ -495,18 +497,16 @@ public class HighLightFillingServiceImpl implements HighLightFillingService {
res.put("_id", searchHit.getId()); res.put("_id", searchHit.getId());
res.put("markPt", "知乎"); res.put("markPt", "知乎");
String insert_at = res.getString("insert_at"); String insert_at = res.getString("insert_at");
if (insert_at != null && !insert_at.equals("") && !insert_at.replaceFirst("000", "").equals("")) { if (insert_at != null && !insert_at.equals("") && !insert_at.replaceFirst("000", "").equals("")) {
Date insertDate = TimeUtil.parseTime(insert_at.replaceFirst("000", ""), Date insertDate = TimeUtil.parseTime(insert_at.replaceFirst("000", ""), "yyyy-MM-dd'T'HH:mm:ss.SSS");
"yyyy-MM-dd'T'HH:mm:ss.SSS");
if (insertDate != null) if (insertDate != null)
res.put("insert_at", insertDate.toString()); res.put("insert_at", insertDate.toString());
} }
String update_at = res.getString("update_at"); String update_at = res.getString("update_at");
if (update_at != null && !update_at.equals("") && !update_at.replaceFirst("000", "").equals("")) { if (update_at != null && !update_at.equals("") && !update_at.replaceFirst("000", "").equals("")) {
Date updateDate = TimeUtil.parseTime(update_at.replaceFirst("000", ""), Date updateDate = TimeUtil.parseTime(update_at.replaceFirst("000", ""), "yyyy-MM-dd'T'HH:mm:ss.SSS");
"yyyy-MM-dd'T'HH:mm:ss.SSS");
if (updateDate != null) if (updateDate != null)
res.put("update_at", updateDate.toString()); res.put("update_at", updateDate.toString());
} }
...@@ -574,14 +574,14 @@ public class HighLightFillingServiceImpl implements HighLightFillingService { ...@@ -574,14 +574,14 @@ public class HighLightFillingServiceImpl implements HighLightFillingService {
} }
if (questionTitle.isEmpty()) { if (questionTitle.isEmpty()) {
questionTitle = res.getString("question_title")+""; questionTitle = res.getString("question_title") + "";
if (questionTitle.length() > 300) { if (questionTitle.length() > 300) {
questionTitle = questionTitle.substring(0, 300); questionTitle = questionTitle.substring(0, 300);
} }
} }
if (questionContent.isEmpty()) { if (questionContent.isEmpty()) {
questionContent = res.getString("question_content")+""; questionContent = res.getString("question_content") + "";
if (questionContent.length() > 300) { if (questionContent.length() > 300) {
questionContent = questionContent.substring(0, 300); questionContent = questionContent.substring(0, 300);
...@@ -589,7 +589,7 @@ public class HighLightFillingServiceImpl implements HighLightFillingService { ...@@ -589,7 +589,7 @@ public class HighLightFillingServiceImpl implements HighLightFillingService {
} }
if (answerContent.isEmpty()) { if (answerContent.isEmpty()) {
answerContent = res.getString("answer_content")+""; answerContent = res.getString("answer_content") + "";
if (answerContent.length() > 300) { if (answerContent.length() > 300) {
answerContent = answerContent.substring(0, 300); answerContent = answerContent.substring(0, 300);
} }
...@@ -599,12 +599,13 @@ public class HighLightFillingServiceImpl implements HighLightFillingService { ...@@ -599,12 +599,13 @@ public class HighLightFillingServiceImpl implements HighLightFillingService {
res.put("question_content", questionContent.replaceAll("<img[^>]*>", "")); res.put("question_content", questionContent.replaceAll("<img[^>]*>", ""));
res.put("answer_content", answerContent.replaceAll("<img[^>]*>", "")); res.put("answer_content", answerContent.replaceAll("<img[^>]*>", ""));
res.put("img", img); res.put("img", img);
if (res.getString("img_url") != null) { if (res.getString("img_url") != null) {
String imgUrl = res.getString("img_url"); String imgUrl = res.getString("img_url");
if (null != imgUrl && "".equals(imgUrl)) { if (null != imgUrl && "".equals(imgUrl)) {
res.put("img", imgUrl);; res.put("img", imgUrl);
;
} }
} }
return res; return res;
...@@ -615,20 +616,24 @@ public class HighLightFillingServiceImpl implements HighLightFillingService { ...@@ -615,20 +616,24 @@ public class HighLightFillingServiceImpl implements HighLightFillingService {
Map<String, HighlightField> highlightFieldsHitMap = searchHit.getHighlightFields(); Map<String, HighlightField> highlightFieldsHitMap = searchHit.getHighlightFields();
res.put("markPt", "视频"); res.put("markPt", "视频");
res.put("_id", searchHit.getId()); res.put("_id", searchHit.getId());
try { try {
String vtime = TimeUtil String vtime = TimeUtil
.parseTime(sourceHitMap.get("time").toString().replaceFirst("000", ""), "yyyy-MM-dd'T'HH:mm:ss") .parseTime(sourceHitMap.get("time").toString().replaceFirst("000", ""), "yyyy-MM-dd'T'HH:mm:ss")
.toString(); .toString();
res.put("time", vtime); res.put("time", vtime);
} catch (Exception e) { } catch (Exception e) {
log.error("videotime为空" + searchHit.getId()); log.error("videotime为空" + searchHit.getId());
} }
String title = ""; String title = "";
String content = "";
Text[] titlelist = highlightFieldsHitMap.containsKey("title") Text[] titlelist = highlightFieldsHitMap.containsKey("title")
? highlightFieldsHitMap.get("title").getFragments() ? highlightFieldsHitMap.get("title").getFragments()
: null; : null;
Text[] contentlist = highlightFieldsHitMap.containsKey("content")
? highlightFieldsHitMap.get("content").getFragments()
: null;
if (titlelist != null && titlelist.length > 0) { if (titlelist != null && titlelist.length > 0) {
for (Text string : titlelist) { for (Text string : titlelist) {
// 通过高亮位置截取过长字符串 // 通过高亮位置截取过长字符串
...@@ -636,12 +641,34 @@ public class HighLightFillingServiceImpl implements HighLightFillingService { ...@@ -636,12 +641,34 @@ public class HighLightFillingServiceImpl implements HighLightFillingService {
title = title + cutStr; title = title + cutStr;
} }
} }
if (contentlist != null && contentlist.length > 0) {
for (Text string : contentlist) {
// 通过高亮位置截取过长字符串
String cutStr = string.string();
if (cutStr.length() > 150) {
int i = cutStr.indexOf("<font");
if (i > 10) {
cutStr = cutStr.substring(i - 10, cutStr.length());
cutStr = "……" + cutStr;
}
}
content = content + cutStr;
}
}
if (content.isEmpty()) {
content = sourceHitMap.containsKey("content") ? sourceHitMap.get("content") + "" : "";
if (content.length() > 300) {
content = content.substring(0, 300);
}
}
if (title.isEmpty()) { if (title.isEmpty()) {
title = String.valueOf(sourceHitMap.get("title")); title = String.valueOf(sourceHitMap.get("title"));
} }
res.put("title", title);
res.put("content", content);
res.put("title", title);
return res; return res;
} }
...@@ -652,9 +679,9 @@ public class HighLightFillingServiceImpl implements HighLightFillingService { ...@@ -652,9 +679,9 @@ public class HighLightFillingServiceImpl implements HighLightFillingService {
res.put("markPt", "网媒"); res.put("markPt", "网媒");
res.put("_id", searchHit.getId()); res.put("_id", searchHit.getId());
try { try {
String mtime=TimeUtil String mtime = TimeUtil
.parseTime(sourceHitMap.get("time").toString().replaceFirst("000", ""), "yyyy-MM-dd'T'HH:mm:ss") .parseTime(sourceHitMap.get("time").toString().replaceFirst("000", ""), "yyyy-MM-dd'T'HH:mm:ss")
.toString(); .toString();
res.put("time", mtime); res.put("time", mtime);
} catch (Exception e) { } catch (Exception e) {
log.error("mediatime为空" + searchHit.getId()); log.error("mediatime为空" + searchHit.getId());
...@@ -699,13 +726,14 @@ public class HighLightFillingServiceImpl implements HighLightFillingService { ...@@ -699,13 +726,14 @@ public class HighLightFillingServiceImpl implements HighLightFillingService {
} }
} }
if (title.isEmpty()) { if (title.isEmpty()) {
title = sourceHitMap.containsKey("title") ? String.valueOf(sourceHitMap.get("title")) : "";; title = sourceHitMap.containsKey("title") ? String.valueOf(sourceHitMap.get("title")) : "";
;
} }
res.put("content", content); res.put("content", content);
res.put("title", title); res.put("title", title);
String source = res.getString("source")+""; String source = res.getString("source") + "";
if ((sourceHitMap.get("pt") + "").equals("微信")) { if ((sourceHitMap.get("pt") + "").equals("微信")) {
HuserInfoWeixin huserInfoWeixin = huserInfoDao.getHuserInfoWeixin(source); HuserInfoWeixin huserInfoWeixin = huserInfoDao.getHuserInfoWeixin(source);
if (huserInfoWeixin != null) { if (huserInfoWeixin != null) {
...@@ -724,5 +752,4 @@ public class HighLightFillingServiceImpl implements HighLightFillingService { ...@@ -724,5 +752,4 @@ public class HighLightFillingServiceImpl implements HighLightFillingService {
return res; return res;
} }
} }
...@@ -10,12 +10,15 @@ import java.util.Locale; ...@@ -10,12 +10,15 @@ import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.TimeZone; import java.util.TimeZone;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.SearchHits;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.zhiwei.messageflow.DirectES4RedisThread;
import com.zhiwei.messageflow.bean.MediaMessage; import com.zhiwei.messageflow.bean.MediaMessage;
import com.zhiwei.messageflow.bean.VideoMessage; import com.zhiwei.messageflow.bean.VideoMessage;
import com.zhiwei.messageflow.bean.WeiboMessage; import com.zhiwei.messageflow.bean.WeiboMessage;
...@@ -29,6 +32,7 @@ import com.zhiwei.messageflow.util.ESQueryUtil; ...@@ -29,6 +32,7 @@ import com.zhiwei.messageflow.util.ESQueryUtil;
@Component @Component
public class NoiseProcessingServiceImpl implements NoiseProcessingService { public class NoiseProcessingServiceImpl implements NoiseProcessingService {
private final static Logger log = LogManager.getLogger(NoiseProcessingServiceImpl.class);
@Autowired @Autowired
private HighLightFillingService highLightFillingService; private HighLightFillingService highLightFillingService;
...@@ -637,7 +641,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService { ...@@ -637,7 +641,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
private boolean isTimeOver(String time) { private boolean isTimeOver(String time) {
long current=System.currentTimeMillis();//当前时间毫秒数 long current=System.currentTimeMillis();//当前时间毫秒数
long zero=current/(1000*3600*24)*(1000*3600*24)-TimeZone.getDefault().getRawOffset();//今天零点零分零秒的毫秒数 long zero=current-(24*3600*1000L);//前一天毫秒数
Date date = new Date(); Date date = new Date();
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
// SimpleDateFormat dateFormat = new SimpleDateFormat("EEE MMM d HH:mm:ss 'CST' // SimpleDateFormat dateFormat = new SimpleDateFormat("EEE MMM d HH:mm:ss 'CST'
...@@ -647,7 +651,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService { ...@@ -647,7 +651,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
try { try {
date = dateFormat.parse(time); date = dateFormat.parse(time);
if ((date.getTime() + (8 * 3600 * 1000L)) < zero) { if ((date.getTime() + (8 * 3600 * 1000L)) < zero) {
System.err.println(date.getTime() + (8 * 3600 * 1000L)+"|"+zero); log.info(date.getTime() + (8 * 3600 * 1000L)+"|"+zero);
return true; return true;
} }
} catch (ParseException e) { } catch (ParseException e) {
...@@ -677,15 +681,15 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService { ...@@ -677,15 +681,15 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
String time = map.get("time") != null ? map.get("time").toString() : null; String time = map.get("time") != null ? map.get("time").toString() : null;
if (isTimeOver(time)) { if (isTimeOver(time)) {
System.err.println("weibo:time:" + time + map.get("id").toString()); log.info("weibo:time:" + time +"\t"+ map.get("id").toString());
return isnoise; return true;
} }
String text = map.get("text") != null ? map.get("text").toString() : null; String text = map.get("text") != null ? map.get("text").toString() : null;
String roottext = map.get("roottext") != null ? map.get("roottext").toString() : null; String roottext = map.get("roottext") != null ? map.get("roottext").toString() : null;
String username = map.get("username") != null ? map.get("username").toString() : null; String username = map.get("username") != null ? map.get("username").toString() : null;
if (null == username) { if (null == username) {
return isnoise; return true;
} }
if (noiseRules != null) { if (noiseRules != null) {
for (NoiseRule n : noiseRules) { for (NoiseRule n : noiseRules) {
...@@ -782,8 +786,8 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService { ...@@ -782,8 +786,8 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
String platform = platformNew.getPlatformName(); String platform = platformNew.getPlatformName();
String time = map.get("created_at") != null ? map.get("created_at").toString() : null; String time = map.get("created_at") != null ? map.get("created_at").toString() : null;
if (isTimeOver(time)) { if (isTimeOver(time)) {
System.err.println("zhihu:time:" + time + map.get("id").toString()); log.info("zhihu:time:" + time +"\t"+ map.get("id").toString());
return isnoise; return true;
} }
String text = map.get("question_title") != null ? map.get("question_title").toString() : null; String text = map.get("question_title") != null ? map.get("question_title").toString() : null;
String roottext = map.get("answer_content") != null ? map.get("answer_content").toString() : null; String roottext = map.get("answer_content") != null ? map.get("answer_content").toString() : null;
...@@ -913,8 +917,8 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService { ...@@ -913,8 +917,8 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
String platform = platformNew.getPlatformName(); String platform = platformNew.getPlatformName();
String time = map.get("time") != null ? map.get("time").toString() : null; String time = map.get("time") != null ? map.get("time").toString() : null;
if (isTimeOver(time)) { if (isTimeOver(time)) {
System.err.println("video:time:" + time + map.get("id").toString()); log.info("video:time:" + time +"\t"+ map.get("id").toString());
return isnoise; return true;
} }
String text = map.get("title") != null ? map.get("title").toString() : null; String text = map.get("title") != null ? map.get("title").toString() : null;
String roottext = null; String roottext = null;
...@@ -1043,15 +1047,15 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService { ...@@ -1043,15 +1047,15 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
String platform = platformNew.getPlatformName(); String platform = platformNew.getPlatformName();
String time = map.get("time") != null ? map.get("time").toString() : null; String time = map.get("time") != null ? map.get("time").toString() : null;
if (isTimeOver(time)) { if (isTimeOver(time)) {
System.err.println("media:time:" + time + map.get("id").toString()); log.info("media:time:" + time +"\t"+ map.get("id").toString());
return isnoise; return true;
} }
String text = map.get("title") != null ? map.get("title").toString() : null; String text = map.get("title") != null ? map.get("title").toString() : null;
String roottext = map.get("content") != null ? map.get("content").toString() : null; String roottext = map.get("content") != null ? map.get("content").toString() : null;
String username = map.get("source") != null ? map.get("source").toString() : null; String username = map.get("source") != null ? map.get("source").toString() : null;
if (null == username) { if (null == username) {
return isnoise; return true;
} }
if (noiseRules != null) { if (noiseRules != null) {
for (NoiseRule n : noiseRules) { for (NoiseRule n : noiseRules) {
......
...@@ -136,7 +136,7 @@ public class RedisPoolAndTools { ...@@ -136,7 +136,7 @@ public class RedisPoolAndTools {
} }
/** /**
* 删除超出上限的数据 * 删除超出上限的数据(弃用
* *
* @param key * @param key
* @param removeIndex * @param removeIndex
...@@ -153,9 +153,26 @@ public class RedisPoolAndTools { ...@@ -153,9 +153,26 @@ public class RedisPoolAndTools {
jedis.zremrangeByRank(key, 0, removeIndex); jedis.zremrangeByRank(key, 0, removeIndex);
returnResource(jedis); returnResource(jedis);
} }
/** /**
* 获取有序集合消息数量 * 删除超出上限的数据
*
* @param key
* @param removeIndex
*/
public void removeDataByName(String key, int removeIndex) {
Jedis jedis = getJedis();
while (true) {
if (null != jedis) {
break;
} else {
jedis = getJedis();
}
}
jedis.zremrangeByRank(key, 0, removeIndex);
returnResource(jedis);
}
/**
* 获取有序集合消息数量(弃用
* *
* @param key * @param key
* @return * @return
...@@ -173,6 +190,25 @@ public class RedisPoolAndTools { ...@@ -173,6 +190,25 @@ public class RedisPoolAndTools {
returnResource(jedis); returnResource(jedis);
return nowCount; return nowCount;
} }
/**
* 获取有序集合消息数量
*
* @param key
* @return
*/
public Long getNowCount(String key) {
Jedis jedis = getJedis();
while (true) {
if (null != jedis) {
break;
} else {
jedis = getJedis();
}
}
Long nowCount = jedis.zcard(key);
returnResource(jedis);
return nowCount;
}
/** /**
* 分页获取redis数据 * 分页获取redis数据
......
...@@ -61,10 +61,10 @@ public class RedisServiceImpl implements RedisService { ...@@ -61,10 +61,10 @@ public class RedisServiceImpl implements RedisService {
/** /**
* 删除超出存储上限的数据 * 删除超出存储上限的数据
*/ */
long nowCount = redisPoolAndTools.getNowCount(redisKey.getBytes());// 当前数据量 long nowCount = redisPoolAndTools.getNowCount(redisKey);// 当前数据量
int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量 int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量
if (maxSize > 0 && nowCount > maxSize) { if (maxSize > 0 && nowCount > maxSize) {
redisPoolAndTools.removeDataByName(redisKey.getBytes(), removeIndex); redisPoolAndTools.removeDataByName(redisKey, removeIndex);
} }
} }
...@@ -84,10 +84,10 @@ public class RedisServiceImpl implements RedisService { ...@@ -84,10 +84,10 @@ public class RedisServiceImpl implements RedisService {
/** /**
* 删除超出存储上限的数据 * 删除超出存储上限的数据
*/ */
long nowCount = redisPoolAndTools.getNowCount(redisKey.getBytes());// 当前数据量 long nowCount = redisPoolAndTools.getNowCount(redisKey);// 当前数据量
int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量 int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量
if (maxSize > 0 && nowCount > maxSize) { if (maxSize > 0 && nowCount > maxSize) {
redisPoolAndTools.removeDataByName(redisKey.getBytes(), removeIndex); redisPoolAndTools.removeDataByName(redisKey, removeIndex);
} }
} }
...@@ -107,10 +107,10 @@ public class RedisServiceImpl implements RedisService { ...@@ -107,10 +107,10 @@ public class RedisServiceImpl implements RedisService {
/** /**
* 删除超出存储上限的数据 * 删除超出存储上限的数据
*/ */
long nowCount = redisPoolAndTools.getNowCount(redisKey.getBytes());// 当前数据量 long nowCount = redisPoolAndTools.getNowCount(redisKey);// 当前数据量
int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量 int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量
if (maxSize > 0 && nowCount > maxSize) { if (maxSize > 0 && nowCount > maxSize) {
redisPoolAndTools.removeDataByName(redisKey.getBytes(), removeIndex); redisPoolAndTools.removeDataByName(redisKey, removeIndex);
} }
} }
...@@ -130,10 +130,10 @@ public class RedisServiceImpl implements RedisService { ...@@ -130,10 +130,10 @@ public class RedisServiceImpl implements RedisService {
/** /**
* 删除超出存储上限的数据 * 删除超出存储上限的数据
*/ */
long nowCount = redisPoolAndTools.getNowCount(redisKey.getBytes());// 当前数据量 long nowCount = redisPoolAndTools.getNowCount(redisKey);// 当前数据量
int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量 int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量
if (maxSize > 0 && nowCount > maxSize) { if (maxSize > 0 && nowCount > maxSize) {
redisPoolAndTools.removeDataByName(redisKey.getBytes(), removeIndex); redisPoolAndTools.removeDataByName(redisKey, removeIndex);
} }
} }
...@@ -148,10 +148,10 @@ public class RedisServiceImpl implements RedisService { ...@@ -148,10 +148,10 @@ public class RedisServiceImpl implements RedisService {
/** /**
* 删除超出存储上限的数据 * 删除超出存储上限的数据
*/ */
long nowCount = redisPoolAndTools.getNowCount(redisKey.getBytes());// 当前数据量 long nowCount = redisPoolAndTools.getNowCount(redisKey);// 当前数据量
int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量 int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量
if (maxSize > 0 && nowCount > maxSize) { if (maxSize > 0 && nowCount > maxSize) {
redisPoolAndTools.removeDataByName(redisKey.getBytes(), removeIndex); redisPoolAndTools.removeDataByName(redisKey, removeIndex);
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment