新增微博话题采集话题贡献者,关于功能

src/main/java/com/zhiwei/searchhotcrawler/bean/WeiBoUser.java

@@ -53,9 +53,13 @@ public class WeiBoUser implements Serializable {
      * 头像地址
      */
     private String profileImageUrl;
+
+    /**
+     * 类型
+     */
+    private String type;
     public WeiBoUser() {
     }
-
     public WeiBoUser(String userId, String attestationMassage, String userName,String topic,Date time,Long followerCount,String profileImageUrl) {
 
         this.id = userId+"_"+HotSearchType.微博热搜.name()+"_"+topic;
@@ -66,6 +70,11 @@ public class WeiBoUser implements Serializable {
         this.time=time;
         this.followerCount=followerCount;
         this.profileImageUrl = profileImageUrl;
-
+    }
+    public WeiBoUser(String userId, String userName,String topic,Date time) {
+        this.userId = userId;
+        this.userName = userName;
+        this.topic=topic;
+        this.time=time;
     }
 }

src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboHotSearchCrawler.java

 package com.zhiwei.searchhotcrawler.crawler;
 
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.stream.Collectors;
 
 import com.alibaba.fastjson.JSON;
+import com.mongodb.client.result.UpdateResult;
 import com.zhiwei.searchhotcrawler.bean.*;
 import com.zhiwei.searchhotcrawler.config.RedisConfig;
 import com.zhiwei.searchhotcrawler.dao.RedisDao;
@@ -17,6 +20,12 @@ import lombok.extern.log4j.Log4j2;
 import okhttp3.Request;
 import okhttp3.Response;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
 import org.bson.Document;
 import org.checkerframework.checker.units.qual.C;
 import org.jsoup.Jsoup;
@@ -37,9 +46,9 @@ import org.springframework.beans.factory.annotation.Autowired;
 import static java.util.Objects.nonNull;
 
 /**
+ * @author hero
  * @ClassName: WeiboHotSearch
  * @Description: 微博实时热搜采集
- * @author hero
  * @date 2017年9月15日 上午10:54:31
  */
 @Log4j2
@@ -113,25 +122,23 @@ public class WeiboHotSearchCrawler {
 //	}
 
 
-
-
     /**
+     * @return void 返回类型
      * @Title: weiboHotSearchByPhoneTest
      * @author hero
      * @Description: TODO(手机端Iphone 微博热搜采集)
-	 * @return void 返回类型
      */
-	public static List<HotSearchList> weiboHotSearchByPhone(Date date){
+    public static List<HotSearchList> weiboHotSearchByPhone(Date date) {
         String url = "https://m.weibo.cn/api/container/getIndex?containerid=106003type%3D25%26t%3D3%26disable_hot%3D1%26filter_type%3Drealtimehot&title=%E5%BE%AE%E5%8D%9A%E7%83%AD%E6%90%9C&extparam=pos%3D0_0%26mi_cid%3D100103%26cate%3D10103%26filter_type%3Drealtimehot%26c_type%3D30&luicode=10000011&lfid=231583";
-		Map<String,String> headerMap = new HashMap<>();
+        Map<String, String> headerMap = new HashMap<>();
         headerMap.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36");
         String htmlBody = null;
         Request request = RequestUtils.wrapGet(url, headerMap);
-		for(int count =0; count<=5; count++){
-			try(Response response = httpBoot.syncCall(request,ProxyHolder.NAT_HEAVY_PROXY)) {
+        for (int count = 0; count <= 5; count++) {
+            try (Response response = httpBoot.syncCall(request, ProxyHolder.NAT_HEAVY_PROXY)) {
                 htmlBody = response.body().string();
             } catch (IOException e) {
-				log.error("解析微博时热搜时出现连接失败",e);
+                log.error("解析微博时热搜时出现连接失败", e);
             }
             List<HotSearchList> result = new ArrayList<HotSearchList>();
             if (StringUtils.isNotBlank(htmlBody) && htmlBody.contains("cards")) {
@@ -143,8 +150,8 @@ public class WeiboHotSearchCrawler {
                     try {
                         JSONObject card = cards.getJSONObject(0);
                         JSONArray cardGroup = card.getJSONArray("card_group");
-							JSONObject topCard =cardGroup.getJSONObject(0);
-							if(!topCard.containsKey("pic")){
+                        JSONObject topCard = cardGroup.getJSONObject(0);
+                        if (!topCard.containsKey("pic")) {
                             rank = 1;
                         }
                         if (Objects.nonNull(cardGroup) && !cardGroup.isEmpty()) {
@@ -162,15 +169,12 @@ public class WeiboHotSearchCrawler {
                                 if (StringUtils.isNotBlank(icon)) {
                                     icon = icon.split("_")[1].split(".png")[0];
                                 }
-									String rankPic =  cardInfo.getString("pic");
-
 //									String id = "http://s.weibo.com/weibo/" + URLCodeUtil.getURLEncode(name, "utf-8") + "&Refer=top";
-									String urlScheme = cardInfo.getString("scheme");
-									HotSearchList hotSearch = new HotSearchList(urlScheme, name, hotCount, hot, rank, HotSearchType.微博热搜.name(), icon, date);
-									hotSearch.setRankPic(rankPic);
+                                String id = cardInfo.getString("scheme");
+                                HotSearchList hotSearch = new HotSearchList(id, name, hotCount, hot, rank, HotSearchType.微博热搜.name(), icon, date);
                                 result.add(hotSearch);
                                 rank++;
-									redisDao.addDataToSet(RedisConfig.WEIBO_HOTSEARCHIDS,name+"_微博热搜");
+                                redisDao.addDataToSet(RedisConfig.WEIBO_HOTSEARCHIDS, name + "_微博热搜");
                             }
                         } else {
                             log.info("card 数据结构为:{}", card);
@@ -194,38 +198,39 @@ public class WeiboHotSearchCrawler {
 
     /**
      * 微博预热榜（实时上升热点采集)
+     *
      * @param date
      * @return
      */
-	public static List<HotSearchList> weiboPreheatSearch(Date date){
+    public static List<HotSearchList> weiboPreheatSearch(Date date) {
         String url = "https://api.weibo.cn/2/guest/page?c=android&s=3d477777&from=10A8395010&gsid=_2AkMoFNQvf8NhqwJRm_gWy2rkbo1_yA7EieKeSCX0JRM3HRl-wT9kqkIltRV6A-gElEGNj31RgrfclQ31YPAf7UBZPBx2&containerid=106003type%3D25%26t%3D3%26disable_hot%3D1%26filter_type%3Drealtimehot";
         String htmlBody = null;
         Request request = RequestUtils.wrapGet(url);
-		try(Response response = httpBoot.syncCall(request,ProxyHolder.NAT_HEAVY_PROXY)) {
+        try (Response response = httpBoot.syncCall(request, ProxyHolder.NAT_HEAVY_PROXY)) {
             htmlBody = response.body().string();
         } catch (IOException e) {
-			log.error("解析微博热搜时出现连接失败",e);
+            log.error("解析微博热搜时出现连接失败", e);
         }
         List<HotSearchList> result = new ArrayList<>();
-		if (StringUtils.isNotBlank(htmlBody) && htmlBody.contains("cards")){
+        if (StringUtils.isNotBlank(htmlBody) && htmlBody.contains("cards")) {
             JSONArray cardArray = JSON.parseObject(htmlBody).getJSONArray("cards");
-			if(cardArray.size() > 1) {
+            if (cardArray.size() > 1) {
                 JSONObject jsonObject = cardArray.getJSONObject(1);
                 if ("实时上升热点".equals(jsonObject.getString("title")) &&
                         jsonObject.containsKey("card_group")) {
                     JSONArray jsonArray = jsonObject.getJSONArray("card_group");
-					for(int i=0; i<jsonArray.size(); i++){
+                    for (int i = 0; i < jsonArray.size(); i++) {
                         JSONObject cardInfo = jsonArray.getJSONObject(i);
                         String name = cardInfo.getString("desc");
                         long hotCount = cardInfo.getIntValue("desc_extr");
                         String weiboUrl = "http://s.weibo.com/weibo/" + URLCodeUtil.getURLEncode(name, "utf-8") + "&Refer=top";
-						HotSearchList hotSearchList = new HotSearchList(weiboUrl,name,hotCount,null,HotSearchType.微博预热榜.name(),date);
+                        HotSearchList hotSearchList = new HotSearchList(weiboUrl, name, hotCount, null, HotSearchType.微博预热榜.name(), date);
                         result.add(hotSearchList);
                     }
                     //根据热度排序，赋值排名
                     result = result.stream().sorted(Comparator.comparing(HotSearchList::getCount).reversed()).collect(Collectors.toList());
-					int rank =1;
-					for(HotSearchList hotSearchList : result){
+                    int rank = 1;
+                    for (HotSearchList hotSearchList : result) {
                         hotSearchList.setRank(rank);
                         rank++;
                     }
@@ -237,16 +242,17 @@ public class WeiboHotSearchCrawler {
 
     /**
      * 微博热搜数据更新导语，阅读量，讨论量
+     *
      * @param document
      * @return
      */
     public static Document weiboUpdate(Document document) {
-		log.info("更新微博热搜{}导语阅读量和讨论量",document.getString("name"));
-		String url = "https://m.weibo.cn/api/container/getIndex?"+ document.getString("url").substring(
-				document.getString("url").indexOf("?")+1,document.getString("url").indexOf("&"));
+        log.info("更新微博热搜{}导语阅读量和讨论量", document.getString("name"));
+        String url = "https://m.weibo.cn/api/container/getIndex?" + document.getString("url").substring(
+                document.getString("url").indexOf("?") + 1, document.getString("url").indexOf("&"));
         String htmlBody = null;
         Request request = RequestUtils.wrapGet(url);
-		for(int count =0; count<=5; count++) {
+        for (int count = 0; count <= 5; count++) {
             try (Response response = httpBoot.syncCall(request, ProxyHolder.NAT_HEAVY_PROXY)) {
                 htmlBody = response.body().string();
             } catch (IOException e) {
@@ -254,14 +260,14 @@ public class WeiboHotSearchCrawler {
             }
             if (htmlBody != null && htmlBody.contains("data")) {
                 JSONObject json = JSONObject.parseObject(htmlBody).getJSONObject("data").getJSONObject("cardlistInfo");
-				List<JSONObject> cardsJsons = (List<JSONObject>)JSONObject.parseObject(htmlBody).getJSONObject("data").get("cards");
-				if(json.containsKey("desc")){
+                List<JSONObject> cardsJsons = (List<JSONObject>) JSONObject.parseObject(htmlBody).getJSONObject("data").get("cards");
+                if (json.containsKey("desc")) {
                     String topicLead = json.getString("desc");
-					if(!"".equals(topicLead)) {
+                    if (!"".equals(topicLead)) {
                         document.put("topicLead", topicLead);
                     }
                 }
-				if(json.containsKey("cardlist_head_cards")){
+                if (json.containsKey("cardlist_head_cards")) {
                     JSONObject readJson = json.getJSONArray("cardlist_head_cards").getJSONObject(0);
                     if (readJson.containsKey("head_data")) {
                         String midText = readJson.getJSONObject("head_data").getString("midtext");
@@ -270,11 +276,11 @@ public class WeiboHotSearchCrawler {
                         String pictureUrl = readJson.getJSONObject("head_data").getString("portrait_url");
                         document.put("readCount", TipsUtils.getHotCount(read));
                         document.put("discussCount", TipsUtils.getHotCount(discussCount));
-						document.put("pictureUrl",pictureUrl);
-						if (readJson.getJSONObject("head_data").containsKey("downtext")){
+                        document.put("pictureUrl", pictureUrl);
+                        if (readJson.getJSONObject("head_data").containsKey("downtext")) {
                             String downtext = readJson.getJSONObject("head_data").getString("downtext");
-							if(!"".equals(downtext)) {
-								document.put("downtext",downtext.replaceAll("主持人：",""));
+                            if (!"".equals(downtext)) {
+                                document.put("downtext", downtext.replaceAll("主持人：", ""));
                             }
                         }
                     }
@@ -282,10 +288,10 @@ public class WeiboHotSearchCrawler {
 
                 try {
                     //解析cards,获取热门微博、人物
-					if (Objects.isNull(weiBoMassageDao)){
+                    if (Objects.isNull(weiBoMassageDao)) {
                         weiBoMassageDao = new WeiBoMassageDao();
                     }
-					if (Objects.isNull(weiBoUserDao)){
+                    if (Objects.isNull(weiBoUserDao)) {
                         weiBoUserDao = new WeiBoUserDao();
                     }
                     for (JSONObject jsonObject : cardsJsons) {
@@ -304,7 +310,7 @@ public class WeiboHotSearchCrawler {
                                     weiBoMassageDao.addWeiBoMassage(weiBoMassage);
                                 }
                                 List<WeiBoUser> weiBoUserList = analysisWeiBoUsers(cardGroup, document.getString("name"));
-								if (!weiBoUserList.isEmpty()){
+                                if (!weiBoUserList.isEmpty()) {
                                     for (int i = 0; i < weiBoUserList.size(); i++) {
                                         weiBoUserDao.addWeiBoUser(weiBoUserList.get(i));
                                     }
@@ -313,7 +319,7 @@ public class WeiboHotSearchCrawler {
                         }
                     }
                 } catch (Exception e) {
-					log.error("解析cards失败,未获得热门微博,人物信息",e);
+                    log.error("解析cards失败,未获得热门微博,人物信息", e);
                 }
                 return document;
             }
@@ -322,6 +328,118 @@ public class WeiboHotSearchCrawler {
     }
 
     /**
+     * 微博热搜数据更新话题贡献者排行,关于
+     *
+     * @param document
+     * @return
+     */
+    public static Document weiboUpdatePC(Document document) {
+        document.getString("name");
+        String topic = document.getString("name");
+        String gb = "#" + topic + "#";
+        String encode = null;
+        try {
+            encode = URLEncoder.encode(gb, "utf-8");
+        } catch (UnsupportedEncodingException e) {
+            log.error("字符解析成URl模式异常", e);
+        }
+        String url = "https://s.weibo.com/weibo?q=" + encode;
+        String htmlBody = null;
+        Request request = RequestUtils.wrapGet(url);
+        try (Response response = httpBoot.syncCall(request, ProxyHolder.NAT_HEAVY_PROXY)) {
+            htmlBody = response.body().string();
+        } catch (IOException e) {
+            log.error("解析微博时热搜时出现连接失败", e);
+        }
+        if (htmlBody != null && htmlBody.contains("m-main")) {
+            Document docm = new Document();
+            try {
+                org.jsoup.nodes.Document documen = Jsoup.parse(htmlBody);
+                //获取贡献者信息
+                try {
+
+                    Elements cardUser = documen.select("div.card-user");
+                    for (Element element : cardUser) {
+                        if (!element.select("div.card-head").text().isEmpty()) {
+                            Elements li = element.select("ul.card-user-list-a").select("li");
+                            if (Objects.nonNull(li)) {
+                                //循环获取话题贡献者相关信息
+                                for (Element eleme : li) {
+                                    String type = "话题贡献者";
+                                    writeUser(eleme, type, topic);
+                                }
+                            }
+                        } else {
+                            Elements li = element.select("ul.card-user-list-a").select("li");
+                            if (Objects.nonNull(li)) {
+
+                                //循环获取话题贡献者相关信息
+                                for (Element eleme : li) {
+                                    String type = "当事人";
+                                    writeUser(eleme, type, topic);
+                                }
+                            }
+                        }
+                    }
+                } catch (Exception e) {
+                    log.error("话题贡献者排行采集异常", e);
+                }
+                Elements dt = documen.select("div.card-about").select("dt");
+                if (Objects.nonNull(dt)) {
+                    //获取微博关于的相关信息
+                    Elements dd = documen.select("div.card-about").select("dd");
+                    Document dtDocument = new Document();
+                    Document ddDocument = new Document();
+                    for (int i = 0; i < dt.size(); i++) {
+                        String dtText = dt.get(i).text().replaceAll("：", "").trim();
+                        dtDocument.put(String.valueOf(i), dtText);
+                    }
+                    for (int i1 = 0; i1 < dd.size(); i1++) {
+                        Elements a = dd.get(i1).select("a");
+                        List<String> str = new ArrayList<>();
+                        for (int b = 0; b < a.size(); b++) {
+                            String text1 = a.get(b).text();
+                            str.add(text1);
+                        }
+                        ddDocument.put(String.valueOf(i1), str);
+                    }
+                    for (int a = 0; a < dt.size(); a++) {
+                        docm.put(dtDocument.getString(String.valueOf(a)), ddDocument.get(String.valueOf(a)));
+                    }
+                }
+                return docm;
+            } catch (Exception e) {
+                log.error("解析微博话题时出现解析错误", e);
+            }
+        }
+
+        return document;
+    }
+    /**
+     * 写入user数据
+     *
+     * @param eleme
+     * @param type
+     */
+    private static void writeUser(Element eleme, String type, String topic) {
+        Date date = new Date();
+        if (Objects.isNull(weiBoUserDao)) {
+            weiBoUserDao = new WeiBoUserDao();
+        }
+        //获取用户名
+        String userName = eleme.select("a.name").text();
+        String attr = eleme.select("span.avator").select("a").first().attr("href");
+        //获取用户id
+        String userId = attr.substring(14);
+        String id = userId + "_" + type + "_" + topic;
+        WeiBoUser weiBoUser = new WeiBoUser(userName, userId, topic, date);
+        weiBoUser.setType(type);
+        weiBoUser.setId(id);
+        weiBoUserDao.addWeiBoUser(weiBoUser);
+    }
+
+
+    /**
      * 解析微博信息
      *
      * @param cardGroup
@@ -353,8 +471,8 @@ public class WeiboHotSearchCrawler {
         //解析weibo人物信息
         Date date = new Date();
         for (int i = 0; i < cardGroup.size(); i++) {
-			if (3==Integer.valueOf(cardGroup.getJSONObject(i).getString("card_type"))) {
-				if (cardGroup.getJSONObject(i).containsKey("users")){
+            if (3 == Integer.valueOf(cardGroup.getJSONObject(i).getString("card_type"))) {
+                if (cardGroup.getJSONObject(i).containsKey("users")) {
                     JSONArray users = cardGroup.getJSONObject(i).getJSONArray("users");
                     for (int i1 = 0; i1 < users.size(); i1++) {
                         //获取用户id
@@ -366,22 +484,22 @@ public class WeiboHotSearchCrawler {
 
                         //获取粉丝数量
                         String followers_count = users.getJSONObject(i1).getString("followers_count");
-						Long followerCount =null;
-						if (!followers_count.contains("万")){
+                        Long followerCount = null;
+                        if (!followers_count.contains("万")) {
                             followerCount = Long.valueOf(followers_count);
-						}else {
+                        } else {
                             String[] split = followers_count.split("万");
-							followerCount = Long.valueOf(split[0])*10000;
+                            followerCount = Long.valueOf(split[0]) * 10000;
                         }
                         //用户头像地址
                         String profileImageUrl = users.getJSONObject(i1).getString("profile_image_url");
-						WeiBoUser weiBoUser = new WeiBoUser(userId, attestationMassage, userName, topic,date,followerCount,profileImageUrl);
+                        WeiBoUser weiBoUser = new WeiBoUser(userId, attestationMassage, userName, topic, date, followerCount, profileImageUrl);
                         weiBoUserList.add(weiBoUser);
                     }
                 }
                 return weiBoUserList;
-			} else if (10==Integer.valueOf(cardGroup.getJSONObject(i).getString("card_type"))) {
-				if (cardGroup.getJSONObject(i).containsKey("user")){
+            } else if (10 == Integer.valueOf(cardGroup.getJSONObject(i).getString("card_type"))) {
+                if (cardGroup.getJSONObject(i).containsKey("user")) {
                     JSONObject user = cardGroup.getJSONObject(i).getJSONObject("user");
                     //获取用户id
                     String userId = user.getString("id");
@@ -391,16 +509,16 @@ public class WeiboHotSearchCrawler {
                     String attestationMassage = user.getString("verified_reason");
                     //获取粉丝数
                     String followers_count = user.getString("followers_count");
-					Long followerCount =null;
-					if (followers_count.contains("万")){
+                    Long followerCount = null;
+                    if (followers_count.contains("万")) {
                         String[] split = followers_count.split("万");
-						followerCount = Long.valueOf(split[0])*10000;
-					}else {
+                        followerCount = Long.valueOf(split[0]) * 10000;
+                    } else {
                         followerCount = Long.valueOf(followers_count);
                     }
                     //用户头像地址
                     String profileImageUrl = user.getString("profile_image_url");
-					WeiBoUser  weiBoUser = new WeiBoUser(userId, attestationMassage, userName, topic,date,followerCount,profileImageUrl);
+                    WeiBoUser weiBoUser = new WeiBoUser(userId, attestationMassage, userName, topic, date, followerCount, profileImageUrl);
                     weiBoUserList.add(weiBoUser);
                 }
                 return weiBoUserList;
@@ -446,11 +564,11 @@ public class WeiboHotSearchCrawler {
 
         //转发数
         String reposts_count = mblog.getString("reposts_count");
-		Long repostCount =null;
-		if (reposts_count.contains("万")){
+        Long repostCount = null;
+        if (reposts_count.contains("万")) {
             String[] split = reposts_count.split("万");
             repostCount = Long.valueOf(split[0]) * 10000;
-		}else {
+        } else {
             repostCount = Long.valueOf(reposts_count);
         }
         Date createTime = null;
@@ -463,12 +581,12 @@ public class WeiboHotSearchCrawler {
 
             createTime = simpleDateFormat.parse(created_at);
             //编辑时间
-			if (mblog.containsKey("edit_at")){
+            if (mblog.containsKey("edit_at")) {
                 String edit_at = mblog.getString("edit_at");
                 editTime = simpleDateFormat.parse(edit_at);
             }
         } catch (ParseException e) {
-			log.error("创建时间和编辑时间解析异常",e);
+            log.error("创建时间和编辑时间解析异常", e);
         }
 
         String mid = mblog.getString("mid");
@@ -492,7 +610,7 @@ public class WeiboHotSearchCrawler {
         }
 
         WeiBoMassage weiBoMassage = new WeiBoMassage(userId, content, userName, mid, createTime, editTime, cardType, showType,
-				repostCount, commentCount, attitudeCount, source, type, topic,profileImageUrl);
+                repostCount, commentCount, attitudeCount, source, type, topic, profileImageUrl);
         //默认不转发为0
         weiBoMassage.setForward(0);
 
@@ -536,12 +654,12 @@ public class WeiboHotSearchCrawler {
                 pictureUrlList.add(picUrl);
             }
         } else if (weiboJson.containsKey("page_info")) {
-			if (weiboJson.getJSONObject("page_info").containsKey("play_count")){
+            if (weiboJson.getJSONObject("page_info").containsKey("play_count")) {
                 String play = weiboJson.getJSONObject("page_info").getString("play_count");
                 if (play.contains("万")) {
                     String[] split = play.split("万");
                     playCount = Long.valueOf(split[0]) * 10000;
-				}else if(play.contains("次")){
+                } else if (play.contains("次")) {
                     String[] split = play.split("次");
                     playCount = Long.valueOf(split[0]);
                 }

src/main/java/com/zhiwei/searchhotcrawler/dao/HotSearchCacheDAO.java

@@ -208,6 +208,17 @@ public class HotSearchCacheDAO  {
                    }
                    if("微博热搜".equals(type)){
                        nowDoc = WeiboHotSearchCrawler.weiboUpdate(nowDoc);
+                       //更新微博话题贡献者,关于功能
+                       Document documentPC = WeiboHotSearchCrawler.weiboUpdatePC(nowDoc);
+                       if (documentPC.containsKey("分类")) {
+                           nowDoc.put("classify",documentPC.get("分类"));
+                       }
+                       if (documentPC.containsKey("地区")) {
+                           nowDoc.put("region", documentPC.get("地区"));
+                       }
+                       if (documentPC.containsKey("标签")) {
+                           nowDoc.put("label", documentPC.get("标签"));
+                       }
                        if(nowDoc.containsKey("topicLead")){
                            nowDoc.put("topicLead", nowDoc.getString("topicLead"));
                        }

src/main/java/com/zhiwei/searchhotcrawler/dao/WeiBoUserDao.java

@@ -32,6 +32,7 @@ public class WeiBoUserDao {
      * @param weiBoUser
      */
     public void addWeiBoUser(WeiBoUser weiBoUser){
+
         try {
             Document document = new Document();
             document.put("_id",weiBoUser.getId());
@@ -42,8 +43,15 @@ public class WeiBoUserDao {
             document.put("userName",weiBoUser.getUserName());
             document.put("topic",weiBoUser.getTopic());
             document.put("time",weiBoUser.getTime());
+            if (Objects.nonNull(weiBoUser.getType())){
+                document.put("type",weiBoUser.getType());
+            }
+            if (Objects.nonNull(weiBoUser.getFollowerCount())){
                 document.put("followerCount",weiBoUser.getFollowerCount());
+            }
+            if (Objects.nonNull(weiBoUser.getProfileImageUrl())){
                 document.put("profileImageUrl",weiBoUser.getProfileImageUrl());
+            }
             try {
                 mongoCollection.insertOne(document);
             } catch (Exception e) {