Skip to content

S
searchhotcrawler
Commit 88b81e78 by chenweitao
Options

评论量讨论量数据类型超限

parent b21d2070
Showing with 53 additions and 53 deletions
src/main/java/com/zhiwei/searchhotcrawler/bean/HotSearchCache.java
......@@ -33,12 +33,12 @@ public class HotSearchCache {
/**
* 最高热搜值
*/
private Integer highestCount;
private Long highestCount;
/**
* 最新热搜热度值
*/
private Integer lastCount;
private Long lastCount;
/**
* 状态（true 为热搜; false为时时上升）
......@@ -83,12 +83,12 @@ public class HotSearchCache {
/**
* 阅读量
*/
private Integer readCount;
private Long readCount;
/**
* 讨论量
*/
private Integer discussCount;
private Long discussCount;
/**
* 话题真假（腾讯较真榜使用)
......@@ -98,7 +98,7 @@ public class HotSearchCache {
/**
* 首次上榜热度
*/
private Integer firstCount;
private Long firstCount;
/** 详情页图片(微博平台) */
private String pictureUrl;
......@@ -113,7 +113,7 @@ public class HotSearchCache {
*/
private String downtext;
public HotSearchCache(String url, String name, String topicLead, Integer highestCount, Integer lastCount, Boolean hot,
public HotSearchCache(String url, String name, String topicLead, Long highestCount, Long lastCount, Boolean hot,
Date startTime, Date endTime, Integer highestRank, Integer lastRank, String type, Integer duration){
this.id = name + "_" + type;
this.url = url;
......@@ -135,21 +135,21 @@ public class HotSearchCache {
public void setRecommend(Boolean recommend) { this.recommend = recommend; }
public Integer getReadCount() { return readCount; }
public Long getReadCount() { return readCount; }
public void setReadCount(Integer readCount) { this.readCount = readCount; }
public void setReadCount(Long readCount) { this.readCount = readCount; }
public Integer getDiscussCount() { return discussCount; }
public Long getDiscussCount() { return discussCount; }
public void setDiscussCount(Integer discussCount) { this.discussCount = discussCount; }
public void setDiscussCount(Long discussCount) { this.discussCount = discussCount; }
public String getTopicLead() { return topicLead; }
public void setTopicLead(String topicLead) { this.topicLead = topicLead; }
public Integer getFirstCount() { return firstCount; }
public Long getFirstCount() { return firstCount; }
public void setFirstCount(Integer firstCount) { this.firstCount = firstCount; }
public void setFirstCount(Long firstCount) { this.firstCount = firstCount; }
public String getPictureUrl() { return pictureUrl; }
......
src/main/java/com/zhiwei/searchhotcrawler/bean/HotSearchList.java
......@@ -42,7 +42,7 @@ public class HotSearchList implements Serializable{
/**
* 热搜量
*/
private Integer count;
private Long count;
/**
* 状态（true 为热搜; false为时时上升）
......@@ -77,7 +77,7 @@ public class HotSearchList implements Serializable{
/**
* 话题讨论量或阅读量
*/
private Integer commentCount;
private Long commentCount;
/**
* 话题真假结果（腾讯较真榜使用)
......@@ -87,12 +87,12 @@ public class HotSearchList implements Serializable{
/**
* 观看数（目前近B站排行榜使用）
*/
private Integer view;
private Long view;
/**
* 弹幕数（目前仅B站排行榜使用）
*/
private Integer barrage;
private Long barrage;
/**
* 图片地址
......@@ -101,7 +101,7 @@ public class HotSearchList implements Serializable{
public HotSearchList(){}
public HotSearchList(String url, String name, Integer count,Boolean hot,Integer rank,String type,String icon,Date date){
public HotSearchList(String url, String name, Long count,Boolean hot,Integer rank,String type,String icon,Date date){
this.id = name + "_" + System.currentTimeMillis() + "_" + type;
this.url = url;
this.name = name;
......@@ -115,7 +115,7 @@ public class HotSearchList implements Serializable{
}
public HotSearchList(String url, String name, Integer count,Integer rank,String type,Date date){
public HotSearchList(String url, String name, Long count,Integer rank,String type,Date date){
this.id = name + "_" + System.currentTimeMillis()+ "_" + type;
this.url = url;
this.name = name;
......@@ -128,7 +128,7 @@ public class HotSearchList implements Serializable{
}
public HotSearchList(String url, String name, Integer count,Integer rank,String type, Integer commentCount, String topicLead,Date date){
public HotSearchList(String url, String name, Long count,Integer rank,String type, Long commentCount, String topicLead,Date date){
this.id = name + "_" + System.currentTimeMillis()+ "_" + type;
this.url = url;
this.name = name;
......@@ -142,7 +142,7 @@ public class HotSearchList implements Serializable{
this.topicLead = topicLead;
}
public HotSearchList(String url, String name, Integer count, Boolean hot,Integer rank, String type, Date date, String icon, String topicResult){
public HotSearchList(String url, String name, Long count, Boolean hot,Integer rank, String type, Date date, String icon, String topicResult){
this.id = name + "_" + System.currentTimeMillis() + "_" + type;
this.url = url;
this.name = name;
......@@ -156,7 +156,7 @@ public class HotSearchList implements Serializable{
this.topicResult = topicResult;
}
public HotSearchList(String url, String name, String topicLead, Integer count, Boolean hot, Date time, Integer rank, String type, Integer view, Integer barrage, String pictureUrl) {
public HotSearchList(String url, String name, String topicLead, Long count, Boolean hot, Date time, Integer rank, String type, Long view, Long barrage, String pictureUrl) {
this.id = name + "_" + System.currentTimeMillis()+ "_" + type;
this.url = url;
this.name = name;
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/BililiCrawler.java
......@@ -52,16 +52,16 @@ public class BililiCrawler {
int rank = i+1;
String name = data.getString("title");
String topicLead = data.getString("desc");
int count = data.getIntValue("score");
long count = data.getLongValue("score");
String bvid = data.getString("bvid");
String pic = data.getString("pic");
String bUrl = "https://www.bilibili.com/video/"+bvid;
Integer view = null;
Integer barrage = null;
Long view = null;
Long barrage = null;
if(data.containsKey("stat")) {
JSONObject stat = data.getJSONObject("stat");
view = stat.getIntValue("view");
barrage = stat.getIntValue("danmaku");
view = stat.getLongValue("view");
barrage = stat.getLongValue("danmaku");
}
HotSearchList hotSearchList = new HotSearchList(bUrl,name,topicLead,count,null,date,rank,HotSearchType.B站排行榜.name(),view,barrage,pic);
hotSearchLists.add(hotSearchList);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/FengHuangSearchCrawler.java
......@@ -47,8 +47,8 @@ public class FengHuangSearchCrawler {
String topicLead = jsonArray.getJSONObject(i).getString("title");
String fenghuangUrl = jsonArray.getJSONObject(i).getJSONObject("link").getString("weburl");
String hotValue = jsonArray.getJSONObject(i).getJSONObject("hotLabel").getString("hotGrade");
Integer count = hotValue.length()>0 ? TipsUtils.getHotCount(hotValue) : 0;
Integer commentCount = jsonArray.getJSONObject(i).getIntValue("commentsall");
Long count = hotValue.length()>0 ? TipsUtils.getHotCount(hotValue) : 0;
Long commentCount = jsonArray.getJSONObject(i).getLongValue("commentsall");
HotSearchList hotSearchList = new HotSearchList(fenghuangUrl,name,count,
rank,HotSearchType.凤凰新闻热榜.name(),commentCount,topicLead,date);
list.add(hotSearchList);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/MaiMaiHotSearchCrawler.java
......@@ -52,7 +52,7 @@ public class MaiMaiHotSearchCrawler {
icon = jsonObject.getJSONObject("hot_type_card").getString("text");
}
String hotValue = jsonArray.getJSONObject(i).getJSONObject("common").getString("hot_info");
Integer count = hotValue.length() > 0 ? TipsUtils.getHotCount(hotValue) : 0;
Long count = hotValue.length() > 0 ? TipsUtils.getHotCount(hotValue) : 0;
HotSearchList hotSearchList = new HotSearchList(maimaiUrl, name, count, null, rank, HotSearchType.脉脉热榜.name(), icon, date);
list.add(hotSearchList);
}
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/SouhuTopicCrawler.java
......@@ -45,7 +45,7 @@ public class SouhuTopicCrawler {
Integer rank = i+1;
String name = dataJson.getJSONObject(i).getJSONObject("eventNewsInfo").getString("title");
String hotValue = dataJson.getJSONObject(i).getString("value");
Integer count = 0;
Long count;
if(hotValue.contains("观点")) {
count = TipsUtils.getHotCount(hotValue.substring(0, hotValue.indexOf("观点")));
}else{
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/TengXunCrawler.java
......@@ -50,17 +50,17 @@ public class TengXunCrawler {
String urlID = dataJson.getJSONObject(i).getString("id");
String tengxunUrl = null;
//String tengxunUrl = "https://view.inews.qq.com/topic/" + dataJson.getJSONObject(i).getString("id");
Integer count = 0;
Long count = null;
String icon = null;
if (dataJson.getJSONObject(i).containsKey("topic")) {
tengxunUrl = "https://view.inews.qq.com/topic/" + urlID;
count = dataJson.getJSONObject(i).getJSONObject("topic").getIntValue("ranking_score");
count = dataJson.getJSONObject(i).getJSONObject("topic").getLongValue("ranking_score");
if (dataJson.getJSONObject(i).getJSONObject("topic").containsKey("rec_icon")) {
icon = dataJson.getJSONObject(i).getJSONObject("topic").getString("rec_icon");
}
} else if (dataJson.getJSONObject(i).containsKey("hotEvent")) {
tengxunUrl = "https://view.inews.qq.com/hotEvent/" + urlID;
count = dataJson.getJSONObject(i).getJSONObject("hotEvent").getIntValue("hotScore");
count = dataJson.getJSONObject(i).getJSONObject("hotEvent").getLongValue("hotScore");
if (dataJson.getJSONObject(i).getJSONObject("hotEvent").containsKey("rec_icon")) {
icon = dataJson.getJSONObject(i).getJSONObject("hotEvent").getString("rec_icon");
}
......@@ -107,7 +107,7 @@ public class TengXunCrawler {
JSONObject jsonObject = jsonArray.getJSONObject(i);
Integer rank = jsonObject.getIntValue("index");
String name = jsonObject.getString("title");
Integer count = jsonObject.getIntValue("score");
Long count = jsonObject.getLongValue("score");
String tengxunUrl = jsonObject.getString("link");
String topicResult = jsonObject.getString("result");
HotSearchList hotSearchList = new HotSearchList(tengxunUrl,name,count,false,rank,HotSearchType.腾讯较真榜.name(),date,null,topicResult);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/ToutiaoHotSearchCrawler.java
......@@ -77,7 +77,7 @@ public class ToutiaoHotSearchCrawler {
String name = word.getString("Title");
// String link = "https://ib.snssdk.com/search/?keyword=" + URLCodeUtil.getURLEncode(name, "utf-8") + "&pd=synthesis&source=trending_list&traffic_source=";
String link = word.getString("Url");
Integer hotCount = word.getInteger("HotValue");
Long hotCount = word.getLongValue("HotValue");
String wordsType = word.getString("Label");
String icon = getIcon(wordsType);
......@@ -170,7 +170,7 @@ public class ToutiaoHotSearchCrawler {
if (Objects.nonNull(elements) && !elements.isEmpty()) {
Element element = elements.first();
String readCount = element.text().replaceAll("阅读", "");
Integer count = TipsUtils.getHotCount(readCount);
Long count = TipsUtils.getHotCount(readCount);
log.info("{},阅读量:{}", hotSearchList.getName(), count);
hotSearchList.setCommentCount(count);
hotSearchListDAO.updateTouTiaoReadCount(hotSearchList);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboHotSearchCrawler.java
......@@ -149,7 +149,7 @@ public class WeiboHotSearchCrawler {
for (int j = 0; j < cardGroup.size(); j++) {
JSONObject cardInfo = cardGroup.getJSONObject(j);
String name = cardInfo.getString("desc");
int hotCount = cardInfo.getIntValue("desc_extr");
long hotCount = cardInfo.getLongValue("desc_extr");
String icon = cardInfo.getString("icon");
if (StringUtils.isNotBlank(icon)) {
icon = icon.split("_")[1].split(".png")[0];
......@@ -205,7 +205,7 @@ public class WeiboHotSearchCrawler {
for(int i=0; i<jsonArray.size(); i++){
JSONObject cardInfo = jsonArray.getJSONObject(i);
String name = cardInfo.getString("desc");
int hotCount = cardInfo.getIntValue("desc_extr");
long hotCount = cardInfo.getIntValue("desc_extr");
String weiboUrl = "http://s.weibo.com/weibo/" + URLCodeUtil.getURLEncode(name, "utf-8") + "&Refer=top";
HotSearchList hotSearchList = new HotSearchList(weiboUrl,name,hotCount,null,HotSearchType.微博预热榜.name(),date);
result.add(hotSearchList);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboTopicCrawler.java
......@@ -169,8 +169,8 @@ public class WeiboTopicCrawler {
String topicName = null;
String url = null;
String description = null;
Integer commentNum = null;
Integer readNum = null;
Long commentNum = null;
Long readNum = null;
String desc2 = null;
for(int i=0; i<cards.size(); i++) {
JSONObject cardGroup = cards.getJSONObject(i);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/XinLangHotSearchCrawler.java
......@@ -57,7 +57,7 @@ public class XinLangHotSearchCrawler {
String name = jsonArray.getJSONObject(i).getString("text");
Integer rank = i + 1;
String hotValue = jsonArray.getJSONObject(i).getString("hotValue");
Integer count = TipsUtils.getHotCount(hotValue);
Long count = TipsUtils.getHotCount(hotValue);
String showTags;
if (jsonArray.getJSONObject(i).containsKey("card")){
JSONArray cardArray = jsonArray.getJSONObject(i).getJSONArray("card");
......@@ -124,7 +124,7 @@ public class XinLangHotSearchCrawler {
Integer rank = i + 1;
String name = dataJson.getJSONObject(i).getString("title");
String xinlangUrl = dataJson.getJSONObject(i).getString("wapurl");
Integer hot = dataJson.getJSONObject(i).getIntValue("hot_value");
Long hot = dataJson.getJSONObject(i).getLongValue("hot_value");
HotSearchList hotSearchList = new HotSearchList(xinlangUrl, name, hot, rank, HotSearchType.新浪热点.name(),date);
hotSearchLists.add(hotSearchList);
}
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/ZhihuChildHotSearchCrawler.java
......@@ -52,7 +52,7 @@ public class ZhihuChildHotSearchCrawler {
Integer rank = i + 1;
String name = jsonObject.getJSONObject("title_area").getString("text");
String hotCountString = jsonObject.getJSONObject("metrics_area").getString("text");
Integer count = TipsUtils.getHotCount(hotCountString.substring(0, hotCountString.indexOf("领域热度")));
Long count = TipsUtils.getHotCount(hotCountString.substring(0, hotCountString.indexOf("领域热度")));
String childUrl = jsonObject.getJSONObject("link").getString("url");
HotSearchList hotSearchList = new HotSearchList(childUrl, name, count, rank, HotSearchType.知乎热搜.name() + typeName + "分类",date);
list.add(hotSearchList);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/ZhihuHotSearchCrawler.java
......@@ -106,7 +106,7 @@ public class ZhihuHotSearchCrawler {
JSONArray dataJson = topSearch.getJSONArray("data");
String link = null;
String displayQuery = null;
Integer hotCount = null;
Long hotCount = null;
String hotText = null;
for (int i = 0; i < dataJson.size(); i++) {
JSONObject data = dataJson.getJSONObject(i).getJSONObject("target");
......@@ -118,12 +118,12 @@ public class ZhihuHotSearchCrawler {
try {
if (hotText.contains("万")) {
hotText = hotText.replaceAll("万.*", "").trim();
hotCount = (int) (Double.parseDouble(hotText) * 10000);
hotCount = (long) (Double.parseDouble(hotText) * 10000);
} else if (hotText.contains("亿")) {
hotText = hotText.replaceAll("亿.*", "").trim();
hotCount = (int) (Double.parseDouble(hotText) * 100000000);
hotCount = (long) (Double.parseDouble(hotText) * 100000000);
} else {
hotCount = Integer.getInteger(hotText);
hotCount = Long.getLong(hotText);
}
} catch (Exception e) {
e.printStackTrace();
......
src/main/java/com/zhiwei/searchhotcrawler/util/TipsUtils.java
......@@ -68,16 +68,16 @@ public class TipsUtils {
* @param hotCountString
* @return
*/
public static Integer getHotCount(String hotCountString){
Integer count;
public static Long getHotCount(String hotCountString){
Long count;
if(hotCountString.contains("万")){
hotCountString = hotCountString.replaceAll("万.*", "").trim();
count = (int)(Double.parseDouble(hotCountString)*10000);
count = (long)(Double.parseDouble(hotCountString)*10000);
}else if(hotCountString.contains("亿")){
hotCountString = hotCountString.replaceAll("亿.*", "").trim();
count = (int)(Double.parseDouble(hotCountString)*100000000);
count = (long)(Double.parseDouble(hotCountString)*100000000);
}else{
count = (int)(Double.parseDouble(hotCountString));
count = (long)(Double.parseDouble(hotCountString));
}
return count;
}
......
src/test/java/weiboTest/WeiboHotSearchTest.java
......@@ -201,7 +201,7 @@ public class WeiboHotSearchTest{
for (int j = 0; j < cardGroup.size(); j++) {
JSONObject cardInfo = cardGroup.getJSONObject(j);
String name = cardInfo.getString("desc");
int hotCount = cardInfo.getIntValue("desc_extr");
long hotCount = cardInfo.getIntValue("desc_extr");
String icon = cardInfo.getString("icon");
if (StringUtils.isNotBlank(icon)) {
icon = icon.split("_")[1].split(".png")[0];
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment