Commit e67a2230 by 马黎滨

百度热搜采集解析延时修复

parent 856995d4
package com.zhiwei.searchhotcrawler.crawler; package com.zhiwei.searchhotcrawler.crawler;
import java.time.Duration;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
...@@ -28,7 +29,7 @@ import com.zhiwei.searchhotcrawler.bean.HotSearchType; ...@@ -28,7 +29,7 @@ import com.zhiwei.searchhotcrawler.bean.HotSearchType;
@Log4j2 @Log4j2
public class BaiDuHotSearchCrawler { public class BaiDuHotSearchCrawler {
private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).build(); private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).connectTimeout(Duration.ofSeconds(60)).build();
/** /**
* @Title: BaiDuHotSearchTest * @Title: BaiDuHotSearchTest
...@@ -101,11 +102,13 @@ public class BaiDuHotSearchCrawler { ...@@ -101,11 +102,13 @@ public class BaiDuHotSearchCrawler {
} }
if (Objects.nonNull(rank)) { if (Objects.nonNull(rank)) {
if(count == 0){ if(count == 0){
log.info(htmlBody);
log.info(hot); log.info(hot);
log.info(element); log.info(element);
} else {
HotSearchList hotSearch = new HotSearchList(everurl, kw, count, rank, HotSearchType.百度热搜.name());
list.add(hotSearch);
} }
HotSearchList hotSearch = new HotSearchList(everurl, kw, count, rank, HotSearchType.百度热搜.name());
list.add(hotSearch);
} }
} catch (Exception e) { } catch (Exception e) {
log.error("解析百度风云榜时出现解析错误", e); log.error("解析百度风云榜时出现解析错误", e);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment