Skip to content

S
searchhotcrawler
Commit d2e5b1cc by leiliangliang
Options

更新微博话题解析,新增用户头像地址

parent b8f83e22
Showing with 34 additions and 12 deletions
src/main/java/com/zhiwei/searchhotcrawler/bean/WeiBoMassage.java
......@@ -88,7 +88,10 @@ public class WeiBoMassage implements Serializable {
* 话题
*/
private String topic;
/**
* 头像地址
*/
private String profileImageUrl;
//是否转发
private Integer forward;
//转发 源微博mid
......@@ -110,7 +113,7 @@ public class WeiBoMassage implements Serializable {
public WeiBoMassage(String userId, String text, String userName, String mid,
Date creatTime, Date editTime, Integer cardType, Integer showType, Long repostCount,
Long commentCount, Long attitudeCount, String source, String type, String topic) {
Long commentCount, Long attitudeCount, String source, String type, String topic, String profileImageUrl) {
this.id =mid+"_"+HotSearchType.微博热搜.name()+"_"+topic;
this.userId = userId;
this.text = text;
......@@ -126,6 +129,8 @@ public class WeiBoMassage implements Serializable {
this.source = source;
this.type = type;
this.topic = topic;
this.profileImageUrl = profileImageUrl;
}
}
src/main/java/com/zhiwei/searchhotcrawler/bean/WeiBoUser.java
......@@ -48,10 +48,15 @@ public class WeiBoUser implements Serializable {
* 粉丝数
*/
private Long followerCount;
/**
* 头像地址
*/
private String profileImageUrl;
public WeiBoUser() {
}
public WeiBoUser(String userId, String attestationMassage, String userName,String topic,Date time,Long followerCount) {
public WeiBoUser(String userId, String attestationMassage, String userName,String topic,Date time,Long followerCount,String profileImageUrl) {
this.id = userId+"_"+HotSearchType.微博热搜.name()+"_"+topic;
this.userId = userId;
......@@ -60,6 +65,7 @@ public class WeiBoUser implements Serializable {
this.topic=topic;
this.time=time;
this.followerCount=followerCount;
this.profileImageUrl = profileImageUrl;
}
}
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboHotSearchCrawler.java
......@@ -370,8 +370,9 @@ public class WeiboHotSearchCrawler {
String[] split = followers_count.split("万");
followerCount = Long.valueOf(split[0])*10000;
}
WeiBoUser weiBoUser = new WeiBoUser(userId, attestationMassage, userName, topic,date,followerCount);
//用户头像地址
String profileImageUrl = users.getJSONObject(i1).getString("profile_image_url");
WeiBoUser weiBoUser = new WeiBoUser(userId, attestationMassage, userName, topic,date,followerCount,profileImageUrl);
weiBoUserList.add(weiBoUser);
}
}
......@@ -394,8 +395,9 @@ public class WeiboHotSearchCrawler {
}else {
followerCount = Long.valueOf(followers_count);
}
WeiBoUser weiBoUser = new WeiBoUser(userId, attestationMassage, userName, topic,date,followerCount);
//用户头像地址
String profileImageUrl = user.getString("profile_image_url");
WeiBoUser weiBoUser = new WeiBoUser(userId, attestationMassage, userName, topic,date,followerCount,profileImageUrl);
weiBoUserList.add(weiBoUser);
}
return weiBoUserList;
......@@ -473,6 +475,8 @@ public class WeiboHotSearchCrawler {
String userName = mblog.getJSONObject("user").getString("screen_name");
//来源
String source = mblog.getString("source");
//用户头像地址
String profileImageUrl = mblog.getJSONObject("user").getString("profile_image_url");
//内容
String content = null;
if (mblog.getString("text").contains("<")) {
......@@ -485,7 +489,7 @@ public class WeiboHotSearchCrawler {
}
WeiBoMassage weiBoMassage = new WeiBoMassage(userId, content, userName, mid, createTime, editTime, cardType, showType,
repostCount, commentCount, attitudeCount, source, type, topic);
repostCount, commentCount, attitudeCount, source, type, topic,profileImageUrl);
//默认不转发为0
weiBoMassage.setForward(0);
......
src/main/java/com/zhiwei/searchhotcrawler/dao/WeiBoMassageDao.java
......@@ -49,6 +49,7 @@ public class WeiBoMassageDao {
document.put("repostCount",weiBoMassage.getRepostCount());
document.put("commentCount",weiBoMassage.getCommentCount());
document.put("attitudeCount",weiBoMassage.getAttitudeCount());
document.put("profileImageUrl",weiBoMassage.getProfileImageUrl());
if (Objects.nonNull(weiBoMassage.getPlayCount())){
document.put("playCount",weiBoMassage.getPlayCount());
}
......
src/main/java/com/zhiwei/searchhotcrawler/dao/WeiBoUserDao.java
......@@ -43,6 +43,7 @@ public class WeiBoUserDao {
document.put("topic",weiBoUser.getTopic());
document.put("time",weiBoUser.getTime());
document.put("followerCount",weiBoUser.getFollowerCount());
document.put("profileImageUrl",weiBoUser.getProfileImageUrl());
try {
mongoCollection.insertOne(document);
} catch (Exception e) {
......
src/test/java/weiboTest/WeiboHotSearchTest.java
......@@ -333,7 +333,9 @@ public class WeiboHotSearchTest {
String[] split = followers_count.split("万");
followerCount = Long.valueOf(split[0])*10000;
}
WeiBoUser weiBoUser = new WeiBoUser(userId, attestationMassage, userName, topic,date,followerCount);
//用户头像地址
String profileImageUrl = users.getJSONObject(i1).getString("profile_image_url");
WeiBoUser weiBoUser = new WeiBoUser(userId, attestationMassage, userName, topic,date,followerCount,profileImageUrl);
weiBoUserList.add(weiBoUser);
}
}
......@@ -356,8 +358,9 @@ public class WeiboHotSearchTest {
}else {
followerCount = Long.valueOf(followers_count);
}
WeiBoUser weiBoUser = new WeiBoUser(userId, attestationMassage, userName, topic,date,followerCount);
//用户头像地址
String profileImageUrl = user.getString("profile_image_url");
WeiBoUser weiBoUser = new WeiBoUser(userId, attestationMassage, userName, topic,date,followerCount,profileImageUrl);
weiBoUserList.add( weiBoUser);
}
......@@ -436,6 +439,8 @@ public class WeiboHotSearchTest {
String userName = mblog.getJSONObject("user").getString("screen_name");
//来源
String source = mblog.getString("source");
//用户头像地址
String profileImageUrl = mblog.getJSONObject("user").getString("profile_image_url");
//内容
String content = null;
if (mblog.getString("text").contains("<")) {
......@@ -448,7 +453,7 @@ public class WeiboHotSearchTest {
}
WeiBoMassage weiBoMassage = new WeiBoMassage(userId, content, userName, mid, createTime, editTime, cardType, showType,
repostCount, commentCount, attitudeCount, source, type, topic);
repostCount, commentCount, attitudeCount, source, type, topic,profileImageUrl);
//默认不转发为0
weiBoMassage.setForward(0);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment