Commit 04c6bdac by chenweitao

Merge branch 'working' into 'master'

修复日志漏洞

See merge request !161
parents 2290d4e2 c51af150
package com.zhiwei.searchhotcrawler.cache; package com.zhiwei.searchhotcrawler.cache;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.zhiwei.tools.tools.ZhiWeiTools; import com.zhiwei.tools.tools.ZhiWeiTools;
......
...@@ -129,7 +129,7 @@ public class BaiDuHotSearchCrawler { ...@@ -129,7 +129,7 @@ public class BaiDuHotSearchCrawler {
String everurl = element.select("td.keyword").select("a.list-title").attr("href"); String everurl = element.select("td.keyword").select("a.list-title").attr("href");
// 获取关键词(String) // 获取关键词(String)
String kw = element.select("td.keyword").select("a.list-title").text(); String kw = element.select("td.keyword").select("a.list-title").text();
// logger.info("关键词:{}", kw); // log.info("关键词:{}", kw);
//从连接中获取正确编码关键词 //从连接中获取正确编码关键词
try { try {
if (!everurl.isEmpty()) { if (!everurl.isEmpty()) {
......
...@@ -9,8 +9,7 @@ import lombok.extern.log4j.Log4j2; ...@@ -9,8 +9,7 @@ import lombok.extern.log4j.Log4j2;
import okhttp3.Request; import okhttp3.Request;
import okhttp3.Response; import okhttp3.Response;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
...@@ -71,7 +70,7 @@ public class DouyinHotSearchCrawler { ...@@ -71,7 +70,7 @@ public class DouyinHotSearchCrawler {
hotValueStr = wl.getString("hot_value"); hotValueStr = wl.getString("hot_value");
Long hotValue = null; Long hotValue = null;
hotValue = Long.valueOf(hotValueStr); hotValue = Long.valueOf(hotValueStr);
// logger.info("热度为:::{}", hot_value); // log.info("热度为:::{}", hot_value);
HotSearchList douyin = new HotSearchList(null, word, hotValue, position, HotSearchType.抖音热搜.name(),date); HotSearchList douyin = new HotSearchList(null, word, hotValue, position, HotSearchType.抖音热搜.name(),date);
list.add(douyin); list.add(douyin);
} }
......
...@@ -99,7 +99,7 @@ public class HotSearch36KrCrawler { ...@@ -99,7 +99,7 @@ public class HotSearch36KrCrawler {
rank++; rank++;
// 获取关键词(String) // 获取关键词(String)
String keyWord = element.select("p.title-wrapper").select("a.article-item-title").text(); String keyWord = element.select("p.title-wrapper").select("a.article-item-title").text();
// logger.info("关键词:{}", kw); // log.info("关键词:{}", kw);
// 获取关键词相关链接everurl(String) // 获取关键词相关链接everurl(String)
String everurl = element.select("p.title-wrapper").select("a.article-item-title").attr("href"); String everurl = element.select("p.title-wrapper").select("a.article-item-title").attr("href");
// 获取搜索指数count(int) // 获取搜索指数count(int)
......
...@@ -13,8 +13,7 @@ import org.jsoup.Jsoup; ...@@ -13,8 +13,7 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.jsoup.select.Elements; import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.zhiwei.crawler.core.HttpBoot; import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.proxy.ProxyHolder; import com.zhiwei.crawler.core.proxy.ProxyHolder;
...@@ -70,7 +69,7 @@ public class SougoHotSearchCrawler { ...@@ -70,7 +69,7 @@ public class SougoHotSearchCrawler {
// 获取关键词(String) // 获取关键词(String)
String kw = element.select("li").select("a").attr("title"); String kw = element.select("li").select("a").attr("title");
// logger.info("关键词:{}", kw); // log.info("关键词:{}", kw);
String everurl = element.select("li").select("a").attr("href"); String everurl = element.select("li").select("a").attr("href");
......
...@@ -13,8 +13,7 @@ import lombok.extern.log4j.Log4j2; ...@@ -13,8 +13,7 @@ import lombok.extern.log4j.Log4j2;
import okhttp3.Request; import okhttp3.Request;
import okhttp3.Response; import okhttp3.Response;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
......
...@@ -18,8 +18,7 @@ import org.apache.commons.lang3.StringUtils; ...@@ -18,8 +18,7 @@ import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.jsoup.select.Elements; import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*; import java.util.*;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
......
...@@ -9,8 +9,7 @@ import java.util.Map; ...@@ -9,8 +9,7 @@ import java.util.Map;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import okhttp3.Request; import okhttp3.Request;
import okhttp3.Response; import okhttp3.Response;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
......
...@@ -99,7 +99,7 @@ public class HotSearch36KrCrawlerTest { ...@@ -99,7 +99,7 @@ public class HotSearch36KrCrawlerTest {
rank++; rank++;
// 获取关键词(String) // 获取关键词(String)
String keyWord = element.select("p.title-wrapper").select("a.article-item-title").text(); String keyWord = element.select("p.title-wrapper").select("a.article-item-title").text();
// logger.info("关键词:{}", kw); // log.info("关键词:{}", kw);
// 获取关键词相关链接everurl(String) // 获取关键词相关链接everurl(String)
String everurl = element.select("p.title-wrapper").select("a.article-item-title").attr("href"); String everurl = element.select("p.title-wrapper").select("a.article-item-title").attr("href");
// 获取搜索指数count(int) // 获取搜索指数count(int)
......
...@@ -12,8 +12,6 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO; ...@@ -12,8 +12,6 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
import com.zhiwei.searchhotcrawler.util.TipsUtils; import com.zhiwei.searchhotcrawler.util.TipsUtils;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import org.bson.Document; import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.mongodb.BasicDBObject; import com.mongodb.BasicDBObject;
import com.mongodb.DBObject; import com.mongodb.DBObject;
......
...@@ -12,8 +12,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO; ...@@ -12,8 +12,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
import com.zhiwei.searchhotcrawler.util.TipsUtils; import com.zhiwei.searchhotcrawler.util.TipsUtils;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import org.bson.Document; import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.mongodb.BasicDBObject; import com.mongodb.BasicDBObject;
import com.mongodb.DBObject; import com.mongodb.DBObject;
......
...@@ -10,8 +10,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchListDAO; ...@@ -10,8 +10,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchListDAO;
import com.zhiwei.searchhotcrawler.util.TipsUtils; import com.zhiwei.searchhotcrawler.util.TipsUtils;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import org.bson.Document; import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.mongodb.BasicDBObject; import com.mongodb.BasicDBObject;
import com.mongodb.DBObject; import com.mongodb.DBObject;
......
...@@ -9,8 +9,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO; ...@@ -9,8 +9,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
import com.zhiwei.searchhotcrawler.util.TipsUtils; import com.zhiwei.searchhotcrawler.util.TipsUtils;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import org.bson.Document; import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.mongodb.BasicDBObject; import com.mongodb.BasicDBObject;
import com.mongodb.DBObject; import com.mongodb.DBObject;
......
...@@ -4,9 +4,9 @@ import com.zhiwei.searchhotcrawler.bean.HotSearchList; ...@@ -4,9 +4,9 @@ import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO; import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
import com.zhiwei.searchhotcrawler.dao.HotSearchListDAO; import com.zhiwei.searchhotcrawler.dao.HotSearchListDAO;
import org.bson.Document; import org.bson.Document;
import org.checkerframework.checker.units.qual.A; import org.apache.logging.log4j.LogManager;
import org.slf4j.Logger; import org.apache.logging.log4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*; import java.util.*;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
...@@ -21,7 +21,7 @@ public class TipsUtils { ...@@ -21,7 +21,7 @@ public class TipsUtils {
private static String key = "a8e26ce3-8aaa-4d3e-bcf6-30b81526050b"; private static String key = "a8e26ce3-8aaa-4d3e-bcf6-30b81526050b";
private Logger logger = LoggerFactory.getLogger(TipsUtils.class); private static final Logger logger = LogManager.getLogger(TipsUtils.class);
private static Map<String,Date> typeTips = new HashMap<>(); private static Map<String,Date> typeTips = new HashMap<>();
......
...@@ -8,8 +8,9 @@ import java.util.Map; ...@@ -8,8 +8,9 @@ import java.util.Map;
import okhttp3.Request; import okhttp3.Request;
import okhttp3.Response; import okhttp3.Response;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
...@@ -21,7 +22,7 @@ import okhttp3.MediaType; ...@@ -21,7 +22,7 @@ import okhttp3.MediaType;
import okhttp3.RequestBody; import okhttp3.RequestBody;
public class WechatCodeUtil { public class WechatCodeUtil {
private static Logger logger = LoggerFactory.getLogger(WechatCodeUtil.class); private static Logger log = LogManager.getLogger(WechatCodeUtil.class);
private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).build(); private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).build();
/** /**
* @Title: getToken * @Title: getToken
...@@ -43,7 +44,7 @@ public class WechatCodeUtil { ...@@ -43,7 +44,7 @@ public class WechatCodeUtil {
result = response.body().string(); result = response.body().string();
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
logger.error("获取微信公众号推送token失败,问题为:::{}", e.fillInStackTrace()); log.error("获取微信公众号推送token失败,问题为:::{}", e.fillInStackTrace());
return null; return null;
} }
if (result != null) { if (result != null) {
...@@ -75,7 +76,7 @@ public class WechatCodeUtil { ...@@ -75,7 +76,7 @@ public class WechatCodeUtil {
try(Response response = httpBoot.syncCall(request)) { try(Response response = httpBoot.syncCall(request)) {
htmlBody = response.body().string(); htmlBody = response.body().string();
} catch (Exception e) { } catch (Exception e) {
logger.error("消息推送失败,错误为::{}",e.fillInStackTrace()); log.error("消息推送失败,错误为::{}",e.fillInStackTrace());
msgid = 0; msgid = 0;
} }
if (StringUtils.isNotBlank(htmlBody)) { if (StringUtils.isNotBlank(htmlBody)) {
...@@ -85,7 +86,7 @@ public class WechatCodeUtil { ...@@ -85,7 +86,7 @@ public class WechatCodeUtil {
msgid = jsonObject.getIntValue("msgid"); msgid = jsonObject.getIntValue("msgid");
} else { } else {
msgid = 0; msgid = 0;
logger.info("消息推送失败,错误为::{}", jsonObject.toString()); log.info("消息推送失败,错误为::{}", jsonObject.toString());
} }
} }
} }
...@@ -117,7 +118,7 @@ public class WechatCodeUtil { ...@@ -117,7 +118,7 @@ public class WechatCodeUtil {
try(Response response = httpBoot.syncCall(request)) { try(Response response = httpBoot.syncCall(request)) {
htmlBody = response.body().string(); htmlBody = response.body().string();
}catch (IOException e){ }catch (IOException e){
logger.error("页面连接获取失败",e); log.error("页面连接获取失败",e);
return null; return null;
} }
if(StringUtils.isNotBlank(htmlBody) && htmlBody.contains("data")) { if(StringUtils.isNotBlank(htmlBody) && htmlBody.contains("data")) {
...@@ -126,12 +127,12 @@ public class WechatCodeUtil { ...@@ -126,12 +127,12 @@ public class WechatCodeUtil {
if(jsonObject.containsKey("data")) { if(jsonObject.containsKey("data")) {
return (List<String>) jsonObject.getJSONObject("data").getObject("openid", List.class); return (List<String>) jsonObject.getJSONObject("data").getObject("openid", List.class);
}else{ }else{
logger.info("拉取用户列表时,出现问题{}", jsonObject); log.info("拉取用户列表时,出现问题{}", jsonObject);
} }
} }
} }
}else{ }else{
logger.info("token 获取失败"); log.info("token 获取失败");
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
...@@ -156,7 +157,7 @@ public class WechatCodeUtil { ...@@ -156,7 +157,7 @@ public class WechatCodeUtil {
try(Response response = httpBoot.syncCall(request)){ try(Response response = httpBoot.syncCall(request)){
htmlBody = response.body().string(); htmlBody = response.body().string();
}catch (IOException e){ }catch (IOException e){
logger.error("页面链接获取失败",e); log.error("页面链接获取失败",e);
return null; return null;
} }
if(StringUtils.isNotBlank(htmlBody) && htmlBody.contains("data")) { if(StringUtils.isNotBlank(htmlBody) && htmlBody.contains("data")) {
...@@ -165,12 +166,12 @@ public class WechatCodeUtil { ...@@ -165,12 +166,12 @@ public class WechatCodeUtil {
if(jsonObject.containsKey("data")) { if(jsonObject.containsKey("data")) {
return (List<String>) jsonObject.getJSONObject("data").getObject("openid", List.class); return (List<String>) jsonObject.getJSONObject("data").getObject("openid", List.class);
}else{ }else{
logger.info("拉取用户列表时,出现问题{}", jsonObject); log.info("拉取用户列表时,出现问题{}", jsonObject);
} }
} }
} }
}else{ }else{
logger.info("token 获取失败"); log.info("token 获取失败");
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
...@@ -199,7 +200,7 @@ public class WechatCodeUtil { ...@@ -199,7 +200,7 @@ public class WechatCodeUtil {
try(Response response = httpBoot.syncCall(request)) { try(Response response = httpBoot.syncCall(request)) {
htmlBody = response.body().string(); htmlBody = response.body().string();
} catch (IOException e) { } catch (IOException e) {
logger.error("获取分组id时出现错误",e.fillInStackTrace()); log.error("获取分组id时出现错误",e.fillInStackTrace());
return null; return null;
} }
if (htmlBody != null) { if (htmlBody != null) {
...@@ -232,7 +233,7 @@ public class WechatCodeUtil { ...@@ -232,7 +233,7 @@ public class WechatCodeUtil {
try(Response response = httpBoot.syncCall(request)) { try(Response response = httpBoot.syncCall(request)) {
htmlBody = response.body().string(); htmlBody = response.body().string();
} catch (IOException e) { } catch (IOException e) {
logger.error("获取分组id时出现错误",e.fillInStackTrace()); log.error("获取分组id时出现错误",e.fillInStackTrace());
return null; return null;
} }
if (htmlBody != null) { if (htmlBody != null) {
...@@ -245,7 +246,7 @@ public class WechatCodeUtil { ...@@ -245,7 +246,7 @@ public class WechatCodeUtil {
resultMap.put(name, id); resultMap.put(name, id);
} }
} else { } else {
logger.info("获取分组id时出现错误,数据为:::{}", htmlBody); log.info("获取分组id时出现错误,数据为:::{}", htmlBody);
} }
} }
return resultMap; return resultMap;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment