Skip to content

S
searchhotcrawler
Commit d1547bb8 by chenweitao
Options

升级爬虫代理

parent eae35e40
Showing with 534 additions and 17 deletions
pom.xml
......@@ -41,7 +41,7 @@
<dependency>
<groupId>com.zhiwei.tools</groupId>
<artifactId>zhiwei-tools</artifactId>
<version>0.1.6-SNAPSHOT</version>
<version>0.4.3-SNAPSHOT</version>
</dependency>
<!--<dependency>-->
<!--<groupId>com.zhiwei.crawler</groupId>-->
......@@ -52,7 +52,12 @@
<dependency>
<groupId>com.zhiwei.http</groupId>
<artifactId>http-boot</artifactId>
<version>0.0.5.9-SNAPSHOT</version>
<version>0.0.8.2-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.zhiwei.crawler</groupId>
<artifactId>proxy-client</artifactId>
<version>1.1.5-SNAPSHOT</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.conscrypt/conscrypt-openjdk-uber -->
<dependency>
......
src/main/java/com/zhiwei/searchhotcrawler/bean/HotSearchList.java
......@@ -9,7 +9,7 @@ package com.zhiwei.searchhotcrawler.bean;
import java.io.Serializable;
import java.util.Date;
import com.zhiwei.tools.timeparse.TimeParse;
import com.zhiwei.searchhotcrawler.util.TimeUtil;
import lombok.Data;
import lombok.ToString;
......@@ -160,7 +160,7 @@ public class HotSearchList implements Serializable{
this.hot = hot;
this.rank = rank;
this.time = date;
this.day = TimeParse.dateFormartString(date, "yyyy-MM-dd");
this.day = TimeUtil.formatTime(date, "yyyy-MM-dd");
this.type = type;
this.icon = icon;
}
......@@ -174,7 +174,7 @@ public class HotSearchList implements Serializable{
this.hot = true;
this.rank = rank;
this.time = date;
this.day = TimeParse.dateFormartString(date, "yyyy-MM-dd");
this.day = TimeUtil.formatTime(date, "yyyy-MM-dd");
this.type = type;
}
......@@ -187,7 +187,7 @@ public class HotSearchList implements Serializable{
this.hot = true;
this.rank = rank;
this.time = date;
this.day = TimeParse.dateFormartString(date, "yyyy-MM-dd");
this.day = TimeUtil.formatTime(date, "yyyy-MM-dd");
this.type = type;
this.commentCount = commentCount;
this.topicLead = topicLead;
......@@ -201,7 +201,7 @@ public class HotSearchList implements Serializable{
this.count = count;
this.rank = rank;
this.time = date;
this.day = TimeParse.dateFormartString(date, "yyyy-MM-dd");
this.day = TimeUtil.formatTime(date, "yyyy-MM-dd");
this.type = type;
this.icon = icon;
this.topicResult = topicResult;
......@@ -216,7 +216,7 @@ public class HotSearchList implements Serializable{
this.hot = hot;
this.time = time;
this.rank = rank;
this.day = TimeParse.dateFormartString(time, "yyyy-MM-dd");
this.day = TimeUtil.formatTime(time, "yyyy-MM-dd");
this.type = type;
this.view = view;
this.barrage = barrage;
......
src/main/java/com/zhiwei/searchhotcrawler/bean/WeiboSuperTopic.java
......@@ -2,15 +2,15 @@ package com.zhiwei.searchhotcrawler.bean;
import java.util.Date;
import com.zhiwei.tools.timeparse.TimeParse;
import com.zhiwei.searchhotcrawler.util.TimeUtil;
import lombok.Data;
import lombok.ToString;
/**
*
*
* @ClassName: WeiboSuperTopic
* @Description: 微博话题
* @author Bewilder ZW
* @author Bewilder ZW
* @date 2019年9月27日 下午3:33:08
*/
@Data
......@@ -72,7 +72,7 @@ public class WeiboSuperTopic {
private Date time;
public WeiboSuperTopic() {}
public WeiboSuperTopic(String url, String topicName, Integer rank, String postNum,
String fensi, String type) {
this.url = url;
......@@ -82,7 +82,7 @@ public class WeiboSuperTopic {
this.fensi = fensi;
this.type = type;
this.time = new Date();
this.day = TimeParse.dateFormartString(new Date(), "yyyy-MM-dd");
this.day = TimeUtil.formatTime(new Date(), "yyyy-MM-dd");
this.id = topicName + "_" + type + "_" + time.getTime();
}
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/SougoHotSearchCrawler.java
......@@ -8,6 +8,7 @@ import com.zhiwei.http.boot.HttpBoot;
import com.zhiwei.http.boot.Response;
import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.util.HeaderTool;
import com.zhiwei.tools.tools.URLCodeUtil;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
......@@ -20,7 +21,6 @@ import org.jsoup.select.Elements;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.tools.httpclient.HeaderTool;
/**
* @ClassName:SougoHotSearch
......@@ -134,4 +134,5 @@ public class SougoHotSearchCrawler {
}
return list;
}
}
src/main/java/com/zhiwei/searchhotcrawler/crawler/ZhihuHotSearchCrawler.java
......@@ -17,7 +17,7 @@ import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.tools.httpclient.HeaderTool;
import com.zhiwei.searchhotcrawler.util.HeaderTool;
import com.zhiwei.tools.tools.URLCodeUtil;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
......
src/main/java/com/zhiwei/searchhotcrawler/timer/DouYinUrlHotSearchRun.java
......@@ -7,7 +7,6 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchListDAO;
import com.zhiwei.searchhotcrawler.util.TipsUtils;
import com.zhiwei.tools.tools.ZhiWeiTools;
import lombok.extern.log4j.Log4j2;
import org.apache.dubbo.common.utils.StringUtils;
import org.bson.Document;
import java.util.ArrayList;
......
src/main/java/com/zhiwei/searchhotcrawler/util/HeaderTool.java 0 → 100644
package com.zhiwei.searchhotcrawler.util;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @author cwt
* @date 2022/4/21 15:43
*/
public class HeaderTool {
private HeaderTool() {
}
public static Map<String, String> getCommonHead() {
Map<String, String> headerMap = new HashMap();
headerMap.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
headerMap.put("Accept-Language", "zh-CN,zh;q=0.9");
headerMap.put("AlexaToolbar-ALX_NS_PH", "AlexaToolbar/alx-4.0");
headerMap.put("Cache-Control", "no-cache");
headerMap.put("Connection", "keep-alive");
headerMap.put("Pragma", "no-cache");
headerMap.put("Proxy-Connection", "keep-alive");
headerMap.put("Upgrade-Insecure-Requests", "1");
headerMap.put("User-Agent", getUserAgent());
return headerMap;
}
private static String getUserAgent() {
List<String> uaList = new ArrayList();
uaList.add("Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50");
uaList.add("Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50");
uaList.add("Mozilla/4.0 (compatible; MSIE 12.0; Windows NT 5.1)");
uaList.add("Mozilla/4.0 (compatible; MSIE 12.0; Windows NT 5.1; Avant Browser)");
uaList.add("Mozilla/4.0 (compatible; MSIE 12.0; Windows NT 5.1; 360SE)");
uaList.add("Mozilla/4.0 (compatible; MSIE 12.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)");
uaList.add("Mozilla/4.0 (compatible; MSIE 12.0; Windows NT 5.1; The World)");
uaList.add("Mozilla/4.0 (compatible; MSIE 12.0; Windows NT 5.1; TencentTraveler 4.0)");
uaList.add("Mozilla/4.0 (compatible; MSIE 12.0; Windows NT 5.1; Maxthon 2.0)");
uaList.add("Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11");
uaList.add("Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1");
uaList.add("Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.167 Safari/537.36");
uaList.add("Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0");
uaList.add("User-Agent\tMozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko");
int index = (int)(Math.random() * (double)uaList.size());
return (String)uaList.get(index);
}
}
src/main/java/com/zhiwei/searchhotcrawler/util/TimeUtil.java 0 → 100644
package com.zhiwei.searchhotcrawler.util;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.temporal.ChronoField;
import java.util.Date;
import static java.util.Objects.isNull;
/**
* @author cwt
* @date 2021/8/19 9:45
*/
public class TimeUtil {
private TimeUtil() {
throw new IllegalStateException("Utility class");
}
private static final DateTimeFormatter DTF_YYYY_MM_DD_HH_MM_SS = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
/** 兼容式格式化工具 */
private static final DateTimeFormatter DTF_ALL = new DateTimeFormatterBuilder()
.appendPattern("yyyy[[-MM][-dd][ HH][:mm][:ss]]")
.parseDefaulting(ChronoField.MONTH_OF_YEAR, 1)
.parseDefaulting(ChronoField.DAY_OF_MONTH, 1)
.parseDefaulting(ChronoField.HOUR_OF_DAY, 0)
.parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)
.parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)
.parseDefaulting(ChronoField.MILLI_OF_SECOND, 0)
.toFormatter();
/**
* 格式化时间 Date -> String
* yyyy-MM-dd HH:mm:ss
* @author cwt
* @param date
* @return String
*/
public static String formatTime(Date date) {
if (isNull(date)) {
return null;
}
Instant instant = date.toInstant();
LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, ZoneId.systemDefault());
return DTF_ALL.format(localDateTime);
}
/**
* 格式化时间 Date -> String
* @author cwt
* @param date
* @param pattern 为空默认为 yyyy-MM-dd HH:mm:ss
* @return java.lang.String
* @date 2021/8/19 9:55
**/
public static String formatTime(Date date, String pattern) {
if (isNull(date)) {
return null;
}
DateTimeFormatter dateTimeFormatter = isNull(pattern) ? DTF_ALL : getDefaultFormatterZero(pattern);
Instant instant = date.toInstant();
LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, ZoneId.systemDefault());
return dateTimeFormatter.format(localDateTime);
}
/**
* 格式化时间 String -> Date
* @author cwt
* @param timeStr 会按格式挑选
* yyyy-MM-dd HH:mm:ss
* yyyy-MM-dd HH:mm
* yyyy-MM-dd HH
* yyyy-MM-dd
* yyyy-MM
* yyyy
* @date 2021/8/23 17:28
* @return java.util.Date
**/
public static Date parseTime(String timeStr) {
if (isNull(timeStr)) {
return null;
}
LocalDateTime localDateTime = LocalDateTime.parse(timeStr, DTF_ALL);
Instant instant = localDateTime.atZone(ZoneId.systemDefault()).toInstant();
return Date.from(instant);
}
/**
* 格式化时间 String -> Date
* @author cwt
* @param timeStr
* @param pattern 为空默认为 yyyy-MM-dd HH:mm:ss
* @return java.lang.String
* @date 2021/8/19 9:55
**/
public static Date parseTime(String timeStr, String pattern) {
if (isNull(timeStr)) {
return null;
}
DateTimeFormatter dateTimeFormatter = isNull(pattern) ? DTF_ALL :
getDefaultFormatterZero(pattern);
LocalDateTime localDateTime = LocalDateTime.parse(timeStr, dateTimeFormatter);
Instant instant = localDateTime.atZone(ZoneId.systemDefault()).toInstant();
return Date.from(instant);
}
/**
* 获取一个赋值初始的 时间格式化对象
* @author cwt
* @return java.time.format.DateTimeFormatter
* @author cwt
* @date 2021/8/23 17:20
**/
private static DateTimeFormatter getDefaultFormatterZero(String pattern) {
return new DateTimeFormatterBuilder()
.appendPattern(pattern)
.parseDefaulting(ChronoField.MONTH_OF_YEAR, 1)
.parseDefaulting(ChronoField.DAY_OF_MONTH, 1)
.parseDefaulting(ChronoField.HOUR_OF_DAY, 0)
.parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)
.parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)
.parseDefaulting(ChronoField.MILLI_OF_SECOND, 0)
.toFormatter();
}
}
src/main/java/com/zhiwei/searchhotcrawler/util/WechatCodeUtil.java
......@@ -13,7 +13,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.tools.httpclient.HeaderTool;
import com.zhiwei.searchhotcrawler.util.HeaderTool;
import okhttp3.MediaType;
import okhttp3.RequestBody;
......
src/test/java/InfoTest/InfoTest.java 0 → 100644
package InfoTest;
import com.mongodb.client.MongoCollection;
import com.zhiwei.http.proxy.ProxyFactory;
import com.zhiwei.proxy.config.SimpleConfig;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.config.DBConfig;
import com.zhiwei.searchhotcrawler.config.ProxyConfig;
import com.zhiwei.searchhotcrawler.crawler.MaiMaiHotSearchCrawler;
import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
import com.zhiwei.searchhotcrawler.dbtemplate.MongoDBTemplate;
import lombok.extern.log4j.Log4j2;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import javax.management.Query;
import java.nio.charset.StandardCharsets;
import java.util.Date;
import java.util.List;
/**
* @author cwt
* @date 2021/10/19 11:38
*/
@Log4j2
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations =
{"classpath:applicationContext.xml"})
public class InfoTest {
@Test
public void testMaimai(){
SimpleConfig simpleConfig = SimpleConfig.builder().registry(ProxyConfig.registry)
.group(ProxyConfig.group).appId(10000013).appName("hotsearch").build();
ProxyFactory.init(simpleConfig);
List<HotSearchList> hotSearchLists = MaiMaiHotSearchCrawler.getMaiMaiHotData(new Date());
for (HotSearchList hotSearchList : hotSearchLists) {
log.info("id长度：{}",hotSearchList.getName().getBytes(StandardCharsets.UTF_8).length);
}
}
@Test
public void indexTest(){
String str = System.currentTimeMillis() + "_" + HotSearchType.脉脉热榜.name();
log.info(str.getBytes(StandardCharsets.UTF_8).length);
String name = "字节跳动，好的很彻底，怀的也很彻底！！！（字节裁员） " +
"字节商业化本地直营少部分城市给赔偿，很多城市的本地直营中心各种劝退员工，疯狂加任务让赚不到钱，威逼利诱等等想让员工离职，不想给赔偿。上级的冷嘲热讽。。。想起了这段时间的捐款，几千万的捐款 其实是想买个好名声，实际做的事情真是坏透了，说白了没有哪一次是想按照劳动法和相关规定给赔偿，ggl是国家政策涉及人也很多，给n加2完全就是想立牌坊。这次部分直营城市赔了，是因为整个城市都裁了，实在是不能压下来，怕有舆论影响。大部分省会直营会留下一部分人，大概百分之30吧，剩下百分之70的人都是各种手段把人搞走，让人自离，真的很恶心，区别对待的背后正好反应了其实就是不遵守劳动法，不想赔钱，赔的每一笔钱都是不得已而为之，甚至听闻连应届毕业生都裁。 无悔入华夏，字节诚负人！！！ 看起来人畜无害，处处标榜正能量的公司坏起来比谁都坏，或许本来骨子里面就是这样";
log.info("name:{}",name.length());
log.info("name.getBytes:{}",name.getBytes(StandardCharsets.UTF_8).length);
// while (name.getBytes(StandardCharsets.UTF_8).length>998){
//
//
// }
}
}
src/test/java/baiduTest/BaiduTest.java 0 → 100644
package baiduTest;
import com.zhiwei.http.boot.HttpBoot;
import com.zhiwei.http.proxy.ProxyFactory;
import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.proxy.config.SimpleConfig;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.config.ProxyConfig;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
import okhttp3.Response;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.*;
/**
* @author cwt
* @date 2021/5/28 21:36
*/
@Log4j2
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations =
{ "classpath:applicationContext.xml" })
public class BaiduTest {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
/**
* 更新数据库中的名字乱码
* @author cwt
* @date 2021/5/31 11:17
* @return void
**/
@Test
public void updateKw(){
//从连接中获取正确编码关键词
// try{
// if (!everurl.isEmpty()){
// kw = URLDecoder.decode(everurl.substring(everurl.indexOf("&wd=")+4).split("&")[0], "GB2312" );
// }
// }catch (Exception e1){
// log.error("解析百度风云榜,地址",e1);
// }
}
@Test
public void test(){
SimpleConfig simpleConfig = SimpleConfig.builder().registry(ProxyConfig.registry)
.group(ProxyConfig.group).appId(10000013).appName("hotsearch").build();
ProxyFactory.init(simpleConfig);
List<HotSearchList> hotSearchLists = baiduHotSearch(new Date());
for (HotSearchList hotSearchList : hotSearchLists) {
System.out.println(hotSearchList.getName());
}
}
@Test
public void testEn(){
String str = "31%CA%A1%B7%DD%D0%C2%D4%F6%B1%BE%CD%C1%C8%B7%D5%EF20%C0%FD%20%BE%F9%D4%DA%B9%E3%B6%AB";
try {
System.out.println(URLDecoder.decode(str, "GBK" ));
String strD = URLDecoder.decode(str, "GB2312" );
System.out.println(strD);
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
}
@Test
public void utfTest(){
String str = "31省份新增本土确诊20例 均在广东";
try {
System.out.println(new String(str.getBytes("ISO-8859-1"),"UTF-8"));
System.out.println(URLEncoder.encode(str,"utf-8"));
String strGBK = URLEncoder.encode(str,"GBK");
System.out.println("GBK: "+strGBK);
System.out.println("GBK解: "+URLDecoder.decode(strGBK,"GBK"));
System.out.println(URLEncoder.encode(str,"GBK"));
System.out.println(URLEncoder.encode(str,"GB2312"));
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
}
/**
* @Title: BaiDuHotSearchTest
* @author hero
* @Description: PC端百度风云榜采集
* @return void 返回类型
*/
public static List<HotSearchList> baiduHotSearch(Date date) {
String url = "http://top.baidu.com/buzz?b=1&fr=topindex";
String htmlBody = null;
// Map<String,String> headers = new HashMap<>();
// headers.put("Content-type","text/html; charset=gb2312");
// Request request = RequestUtils.wrapGet(url, HeadersUtils.convertRepeatably(headers, Charset.forName("gb2312")));
Request request = RequestUtils.wrapGet(url);
// try(Response response = httpBoot.syncCall(request, ProxySupplier.NAT_HEAVY_PROXY)) {
// htmlBody = response.body().string();
// } catch (Exception e) {
// log.error("解析百度风云榜时出现解析错误,页面结构有问题", e);
// }
// if (htmlBody != null && htmlBody.contains("mainBody")) {
// return ansysData(htmlBody,date);
// } else {
// log.info("解析百度风云榜时出现解析错误,页面结构有问题");
// }
return Collections.emptyList();
}
/**
* 解析数据
* @param htmlBody
* @return
*/
private static List<HotSearchList> ansysData(String htmlBody,Date date){
List<HotSearchList> list = new ArrayList<>();
try {
Document document = Jsoup.parse(htmlBody);
Elements elements = document.select("table.list-table").select("tr");
if (Objects.nonNull(elements) && !elements.isEmpty()) {
elements.forEach(element -> {
try {
// 获取排名rank
String rankStr = null;
// 根据网页标签，给rankStr做判断
if (!element.select("td.first").select("span.num-top").isEmpty()) {
rankStr = element.select("td.first").select("span.num-top").text();
} else if (!element.select("td.first").select("span.num-normal").isEmpty()) {
rankStr = element.select("td.first").select("span.num-normal").text();
}
Integer rank = null;
// 判断rankStr是否为空
if (StringUtils.isNoneBlank(rankStr)) {
rank = Integer.valueOf(rankStr);
}
// 获取关键词相关链接everurl(String)
String everurl = element.select("td.keyword").select("a.list-title").attr("href");
// 获取关键词(String)
String kw = element.select("td.keyword").select("a.list-title").text();
// System.out.println(element.select("td.keyword"));
//从连接中获取正确编码关键词
try{
System.out.println("everurl:"+everurl);
if (!everurl.isEmpty()){
kw = URLDecoder.decode(everurl.substring(everurl.indexOf("&wd=")+4).split("&")[0], "GB2312" );
}
System.out.println("kw:"+kw);
}catch (Exception e1){
log.error("解析百度风云榜,地址",e1);
}
// 获取搜索指数count（int）
String hot = null;
// 判断热度值所在的规则是否为null
if (!element.select("td.last").select("span.icon-fall").isEmpty()) {
hot = element.select("td.last").select("span.icon-fall").text();
} else if (!element.select("td.last").select("span.icon-rise").isEmpty()) {
hot = element.select("td.last").select("span.icon-rise").text();
}
else if (!element.select("td.last").select("span.icon-fair").isEmpty()) {
hot = element.select("td.last").select("span.icon-fair").text();
}
long count = 0;
// 判断hot是否为空
if (StringUtils.isNotBlank(hot)) {
count = Integer.valueOf(hot);
}
if (Objects.nonNull(rank)) {
if(count == 0){
log.info(htmlBody);
log.info(hot);
log.info(element);
} else {
HotSearchList hotSearch = new HotSearchList(everurl, kw, count, rank, HotSearchType.百度热搜.name(),date);
list.add(hotSearch);
}
}
} catch (Exception e) {
log.error("解析百度风云榜时出现解析错误", e);
}
});
}
} catch (Exception e) {
log.error("解析百度风云榜时出现解析错误,数据不是json结构", e);
}
return list;
}
}
src/test/java/baiduTest/StrTest.java 0 → 100644
package baiduTest;
import lombok.extern.log4j.Log4j2;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
* @author cwt
* @date 2021/9/9 17:08
*/
@Log4j2
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations =
{ "classpath:applicationContext.xml" })
public class StrTest {
@Test
public void rTest(){
String str = "电影 477113";
String[] strs = str.split(" ");
if (strs.length>1){
System.out.println("标签："+ strs[0].trim());
System.out.println("热度："+ strs[1].trim());
}else {
System.out.println("热度："+ strs[0].trim());
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment