Commit 2247b2af by leiliangliang

增加脉脉话题重试次数

parent 8812fe6a
......@@ -29,13 +29,16 @@ public class MaiMaiTopicCrawler {
String url = "https://open.taou.com/maimai/feed/v6/hot_list_entry/feeds?page_version=2&tab_id=topic&version=6.2.34&u=232258287&access_token=1.4c82e8ad6d6b4e03262a48f334dea336";
String htmlBody = null;
Request request = RequestUtils.wrapGet(url);
for (int x = 0; x <= 2; x++) {
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("脉脉话题页面连接异常...", cause);
continue;
}else {
htmlBody = response.bodyString();
}
try {
if (StringUtils.isNotBlank(htmlBody) && htmlBody.contains("topics")) {
JSONArray jsonArray = JSONObject.parseObject(htmlBody).getJSONArray("topics");
if (jsonArray != null) {
......@@ -50,6 +53,14 @@ public class MaiMaiTopicCrawler {
list.add(hotSearchList);
}
}
}else {
continue;
}
} catch (Exception e) {
log.info("脉脉话题解析异常",e);
continue;
}
break;
}
log.info("{}, 此轮脉脉话题采集到的数据量为:{}", new Date(), list != null ? list.size() : 0);
log.info("脉脉话题采集结束");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment