Skip to content

S
searchhotcrawler
Commit d68f3ced by chenweitao
Options

Merge branch 'mlbWork' into 'master' 

Mlb work

See merge request !74
parents de80cbe7 529937a4
Showing with 46 additions and 52 deletions
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboTopicCrawler.java
......@@ -132,8 +132,8 @@ public class WeiboTopicCrawler {
*/
public static List<HotSearchList> startCrawlerByPhone(Date date){
List<HotSearchList> topicList = new ArrayList<>();
for(int page=1; page<=6; page++){
String pageUrl = "https://m.weibo.cn/api/container/getIndex?containerid=231648_-_2&page=" + page;
for(int page=1; page<=3; page++){
String pageUrl = "https://api.weibo.cn/2/page?st_bottom_bar_new_style_enable=1&c=android&s=34dc160d&from=10A9295010&gsid=_2A25NH7inDeRxGeNH4lUX9ifIzTWIHXVvjUtvrDV6PUJbkdANLRjfkWpNSk7RXJ9vYwBfAr66TNj0zcFmOBPKZDuI&containerid=231648_-_4&page=" + page;
Request request = RequestUtils.wrapGet(pageUrl);
String htmlBody = null;
//重试三次
......@@ -144,7 +144,7 @@ public class WeiboTopicCrawler {
log.error("下载榜单列表页面时出现错误，错误为:{}", e);
continue;
}
if (StringUtils.isNotBlank(htmlBody) && htmlBody.contains("data")) {
if (StringUtils.isNotBlank(htmlBody)) {
topicList.addAll(parseTopicHtml(htmlBody,date));
break;
} else {
......@@ -158,58 +158,52 @@ public class WeiboTopicCrawler {
private static List<HotSearchList> parseTopicHtml(String htmlBody,Date date) {
try {
JSONArray jsonArray = JSONObject.parseObject(htmlBody).getJSONObject("data").getJSONArray("cards");
JSONArray jsonArray = JSONObject.parseObject(htmlBody).getJSONArray("cards");
if(Objects.nonNull(jsonArray) && !jsonArray.isEmpty()) {
JSONArray cards = jsonArray.getJSONObject(0).getJSONArray("card_group");
List<HotSearchList> topicList = new ArrayList<>();
Integer rank = null;
String topicName = null;
String url = null;
String description = null;
Integer commentNum = null;
Integer readNum = null;
String desc2 = null;
for(int i=0; i<cards.size(); i++) {
JSONObject cardGroup = cards.getJSONObject(i);
rank = cardGroup.getInteger("top_mark_text");
topicName = cardGroup.getString("title_sub");
url = "https://s.weibo.com/weibo?q="+ URLCodeUtil.getURLEncode(topicName, "utf-8");
description = null;
if(cardGroup.containsKey("card_expand")){
description = cardGroup.getJSONObject("card_expand").getString("content");
for (int j=0; j< jsonArray.size(); j++){
JSONObject card = jsonArray.getJSONObject(j);
if(card.containsKey("card_group")){
JSONArray cards = card.getJSONArray("card_group");
List<HotSearchList> topicList = new ArrayList<>();
Integer rank = null;
String topicName = null;
String url = null;
String description = null;
Integer commentNum = null;
Integer readNum = null;
String desc2 = null;
for(int i=0; i<cards.size(); i++) {
JSONObject cardGroup = cards.getJSONObject(i);
rank = cardGroup.getInteger("top_mark_text");
topicName = cardGroup.getString("title_sub");
url = "https://s.weibo.com/weibo?q="+ URLCodeUtil.getURLEncode(topicName, "utf-8");
description = null;
if(cardGroup.containsKey("card_expand")){
description = cardGroup.getJSONObject("card_expand").getString("content");
}
desc2 = cardGroup.getString("desc");
String commentNumStr = desc2.replaceAll("讨论.*", "").trim();
String readNumStr = desc2.replaceAll(".*讨论|阅读", "").trim();
try {
commentNum = TipsUtils.getHotCount(commentNumStr);
readNum = TipsUtils.getHotCount(readNumStr);
}catch (Exception e){
e.printStackTrace();
}
HotSearchList topic = new HotSearchList(url, topicName, readNum, rank, HotSearchType.微博话题.name(), commentNum, description,date);
if(cardGroup.containsKey("title_flag_pic")){
String titlePic = cardGroup.getString("title_flag_pic");
if(titlePic.contains("new")){
topic.setIcon("新");
}else if(titlePic.contains("hot")){
topic.setIcon("热");
}
}
topicList.add(topic);
}
return topicList;
}
desc2 = cardGroup.getString("desc");
String commentNumStr = desc2.replaceAll("讨论.*", "").trim();
String readNumStr = desc2.replaceAll(".*讨论|阅读", "").trim();
try {
commentNum = TipsUtils.getHotCount(commentNumStr);
readNum = TipsUtils.getHotCount(readNumStr);
// if(commentNumStr.contains("万")){
// commentNumStr = commentNumStr.replaceAll("万", "");
// commentNum = (int)(Double.parseDouble(commentNumStr)*10000);
// }else if(commentNumStr.contains("亿")){
// commentNumStr = commentNumStr.replaceAll("亿", "");
// commentNum = (int)(Double.parseDouble(commentNumStr)*10000000);
// }else{
// commentNum = Integer.getInteger(commentNumStr);
// }
//
// if(readNumStr.contains("万")){
// readNumStr = readNumStr.replaceAll("万", "");
// readNum = (int)(Double.parseDouble(readNumStr)*10000);
// }else if(readNumStr.contains("亿")){
// readNumStr = readNumStr.replaceAll("亿", "");
// readNum = (int)(Double.parseDouble(readNumStr)*10000000);
// }else{
// readNum = Integer.getInteger(readNumStr);
// }
}catch (Exception e){
e.printStackTrace();
}
HotSearchList topic = new HotSearchList(url, topicName, readNum, rank, HotSearchType.微博话题.name(), commentNum, description,date);
topicList.add(topic);
}
return topicList;
}else{
// log.info("html:{}",htmlBody);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment