Commit d7da84ad by [zhangzhiwei]

添加今日头条账号认证类型及是否认证

parent 9db65425
......@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.zhiwei</groupId>
<artifactId>toutiao</artifactId>
<version>0.3.5-SNAPSHOT</version>
<version>0.3.6-SNAPSHOT</version>
<dependencies>
<dependency>
......
......@@ -15,168 +15,201 @@ public class TouTiaoAccount implements Serializable{
public String id; //主键 帐号id
public Long user_id; //帐号id
public Long userId; //帐号id
public String name; //帐号昵称
public Long media_id; //未知
public Long mediaId; //未知
public String description; //描述
public Integer user_verified; //是否认证 (0,不是;1 是)
public Integer userVerified; //是否认证 (0,不是;1 是)
public String verify_content; //认证原因
public String verifyContent; //认证原因
public Integer follow_count; //粉丝数
public Integer followCount; //粉丝数
public Integer friend_count; //关注数
public Integer friendCount; //关注数
public String img_url; //头像地址
public String imgUrl; //头像地址
public Date create_time; //账号注册时间
public Date createTime; //账号注册时间
public String gender; //性别
public String user_type; //用户类型
public Integer userType; //用户类型
public Integer authType; //用户认证类型
public Boolean isDv; //是否加V
@Override
public String toString(){
return "new TouTiaoAccount["
+ "id = " + id
+ ", user_id = " + user_id
+ ", userId = " + userId
+ ", name = " + name
+ ", media_id = " + media_id
+ ", userId = " + userId
+ ", description = " + description
+ ", user_verified = " + user_verified
+ ", verify_content = " + verify_content
+ ", follow_count = " + follow_count
+ ", friend_count = " + friend_count
+ ", img_url = " + img_url
+ ", create_time = " + create_time
+ ", userVerified = " + userVerified
+ ", verifyContent = " + verifyContent
+ ", followCount = " + followCount
+ ", friendCount = " + friendCount
+ ", imgUrl = " + imgUrl
+ ", createTime = " + createTime
+ ", gender = " + gender
+ ", user_type = " + user_type
+ ", userType = " + userType
+ ", authType = " + authType
+ ", isDv = " + isDv
+ "]";
}
public TouTiaoAccount(){}
public TouTiaoAccount(Long user_id,String name, Long media_id, String description,Integer user_verified
,String verify_content, Integer follow_count, String img_url, Date create_time, String gender
,String user_type){
this.id = user_id+"";
this.user_id = user_id;
public TouTiaoAccount(Long userId,String name, Long mediaId, String description,Integer userVerified
,String verifyContent, Integer followCount, String imgUrl, Date createTime, String gender
,Integer userType, Integer authType, Boolean isDv){
this.id = String.valueOf(userId);
this.userId = userId;
this.name = name;
this.media_id = media_id;
this.mediaId = mediaId;
this.description = description;
this.user_verified = user_verified;
this.verify_content = verify_content;
this.follow_count = follow_count;
this.img_url = img_url;
this.create_time = create_time;
this.userVerified = userVerified;
this.verifyContent = verifyContent;
this.followCount = followCount;
this.imgUrl = imgUrl;
this.createTime = createTime;
this. gender = gender;
this.user_type = user_type;
this.userType = userType;
this.authType = authType;
this.isDv = isDv;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public Long getUser_id() {
return user_id;
public Long getUserId() {
return userId;
}
public void setUser_id(Long user_id) {
this.user_id = user_id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Long getMedia_id() {
return media_id;
public Long getMediaId() {
return mediaId;
}
public void setMedia_id(Long media_id) {
this.media_id = media_id;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
public Integer getUserVerified() {
return userVerified;
}
public Integer getUser_verified() {
return user_verified;
public String getVerifyContent() {
return verifyContent;
}
public void setUser_verified(Integer user_verified) {
this.user_verified = user_verified;
public Integer getFollowCount() {
return followCount;
}
public Integer getFollow_count() {
return follow_count;
public Integer getFriendCount() {
return friendCount;
}
public void setFollow_count(Integer follow_count) {
this.follow_count = follow_count;
public String getImgUrl() {
return imgUrl;
}
public String getVerify_content() {
return verify_content;
public Date getCreateTime() {
return createTime;
}
public void setVerify_content(String verify_content) {
this.verify_content = verify_content;
public String getGender() {
return gender;
}
public String getImg_url() {
return img_url;
public Integer getUserType() {
return userType;
}
public void setImg_url(String img_url) {
this.img_url = img_url;
public Integer getAuthType() {
return authType;
}
public Date getCreate_time() {
return create_time;
public void setId(String id) {
this.id = id;
}
public void setCreate_time(Date create_time) {
this.create_time = create_time;
public void setUserId(Long userId) {
this.userId = userId;
}
public String getGender() {
return gender;
public void setName(String name) {
this.name = name;
}
public void setMediaId(Long mediaId) {
this.mediaId = mediaId;
}
public void setDescription(String description) {
this.description = description;
}
public void setUserVerified(Integer userVerified) {
this.userVerified = userVerified;
}
public void setVerifyContent(String verifyContent) {
this.verifyContent = verifyContent;
}
public void setFollowCount(Integer followCount) {
this.followCount = followCount;
}
public void setFriendCount(Integer friendCount) {
this.friendCount = friendCount;
}
public int getFriend_count() {
return friend_count;
public void setImgUrl(String imgUrl) {
this.imgUrl = imgUrl;
}
public void setFriend_count(int friend_count) {
this.friend_count = friend_count;
public void setCreateTime(Date createTime) {
this.createTime = createTime;
}
......@@ -185,18 +218,23 @@ public class TouTiaoAccount implements Serializable{
}
public String getUser_type() {
return user_type;
public void setUserType(Integer userType) {
this.userType = userType;
}
public void setUser_type(String user_type) {
this.user_type = user_type;
public void setAuthType(Integer authType) {
this.authType = authType;
}
public Boolean getIsDv() {
return isDv;
}
public void setIsDv(Boolean isDv) {
this.isDv = isDv;
}
}
......@@ -2,6 +2,7 @@ package com.zhiwei.toutiao.parse;
import java.net.Proxy;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
......@@ -9,6 +10,7 @@ import java.util.Objects;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.util.Strings;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
......@@ -47,7 +49,7 @@ public class TouTiaoAccountParse {
public static TouTiaoAccount getTouTiaoAccountInfoByName(String name, Proxy proxy){
String url = "https://www.toutiao.com/api/search/content/?aid=24&app_name=web_search&offset=0&format=json&keyword="+URLCodeUtil.getURLEncode(name, "utf-8")+"&autoload=true&count=20&en_qc=1&cur_tab=4&from=media&pd=user&timestamp="+System.currentTimeMillis();
headerMap = Tools.getTouTiaoHeader();
headerMap.put("cookie", "s_v_web_id=58999cab9d5be664e02f961e52e59269");
headerMap.put("cookie", "s_v_web_id=" + Signature.parseStrToMd5L32(name));
TouTiaoAccount tta = null;
try {
String htmlBody = downloadHtml(url, proxy, headerMap);
......@@ -63,28 +65,26 @@ public class TouTiaoAccountParse {
} catch (Exception e) {
e.fillInStackTrace();
logger.error("获取今日头条帐号数据连接超时 {}", e);
return null;
}
return tta;
}
public static TouTiaoAccount getTouTiaoAccountInfoByUserId(String user_id, Proxy proxy){
String url = "https://www.toutiao.com/c/user/"+user_id+"/";
public static TouTiaoAccount getTouTiaoAccountInfoByUserId(String userId, Proxy proxy){
String url = "https://www.toutiao.com/c/user/"+ userId +"/";
headerMap = Tools.getTouTiaoHeader();
headerMap.put("cookie", "s_v_web_id=58999cab9d5be664e02f961e52e59269");
headerMap.put("cookie", "s_v_web_id="+ Signature.parseStrToMd5L32(userId));
TouTiaoAccount tta = null;
try {
String htmlBody = downloadHtml(url, proxy, headerMap);
if(htmlBody != null){
tta = parseAccountByUserId(htmlBody, user_id, proxy);
tta = parseAccountByUserId(htmlBody, userId, proxy);
}
} catch (Exception e) {
e.fillInStackTrace();
logger.error("获取今日头条帐号数据连接超时", e);
return null;
}
return tta;
}
......@@ -108,7 +108,6 @@ public class TouTiaoAccountParse {
while(f){
String url = "https://www.toutiao.com/api/search/content/?aid=24&app_name=web_search&offset="+page*20+"&format=json&keyword="+URLCodeUtil.getURLEncode(word, "utf-8")+"&autoload=true&count=20&en_qc=1&cur_tab=4&from=media&pd=user";
headerMap = Tools.getTouTiaoHeader();
System.out.println(url);
try {
String htmlBody = downloadHtml(url, proxy, headerMap);
if(htmlBody != null){
......@@ -146,7 +145,6 @@ public class TouTiaoAccountParse {
int page = 0;
while(more){
String url = "http://it-hl.snssdk.com/user/relation/following/v2/?user_id="+userid+"&device_id=54560738994&cursor=&iid=53238029655&offset="+page*50+"&count=50&ts="+System.currentTimeMillis()/1000;
System.out.println(url);
page++;
headerMap = Tools.getTouTiaoHeader();
headerMap.put("User-Agent", "Dalvik/2.1.0 (Linux; U; Android 8.1.0; MI 8 MIUI/V10.0.11.0.OEACNFH) NewsArticle/7.0.1 cronet/TTNetVersion:pre_blink_merge-277498-gd2bb364e 2018-08-24");
......@@ -169,7 +167,6 @@ public class TouTiaoAccountParse {
} catch (Exception e) {
logger.error("获取今日头条帐号数据连接超时", e.fillInStackTrace());
more = false;
return null;
}
}
return ttaList;
......@@ -179,11 +176,10 @@ public class TouTiaoAccountParse {
public static List<TouTiaoAccount> getFriendsList(String userid, ProxyHolder proxy){
List<TouTiaoAccount> ttaList = new ArrayList<>();
Signature signature = new Signature(userid, "0");
String _signature = signature.getSignature();
String signatureStr = signature.getSignature();
boolean more = true;
while(more){
String url = "https://www.toutiao.com/c/user/following/?user_id="+userid+"&cursor=0&count=100&_signature="+_signature;
System.out.println(url);
String url = "https://www.toutiao.com/c/user/following/?user_id="+userid+"&cursor=0&count=100&_signature="+ signatureStr;
headerMap = Tools.getTouTiaoHeader();
headerMap.put("referer", "ihttps://www.toutiao.com/c/user/relation/"+ userid +"/?tab=following");
headerMap.put("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36");
......@@ -202,12 +198,10 @@ public class TouTiaoAccountParse {
}
}else{
more = false;
continue;
}
} catch (Exception e) {
logger.error("获取今日头条帐号数据连接超时", e.fillInStackTrace());
more = false;
continue;
}
}
}
......@@ -230,58 +224,63 @@ public class TouTiaoAccountParse {
}
JSONObject json = JSONObject.parseObject(htmlBody);
JSONArray jsonArray = json.getJSONArray("data");
Long user_id = null;
Long userId = null;
String name = null;
Long media_id = -1L;
Long mediaId = null;
String description = null;
Integer user_verified = -1;
String verify_content = null;
Integer follow_count = -1;
Date create_time = null;
String img_url = null;
Integer userVerified = 0;
String verifyContent = null;
Integer followCount = null;
Date createTime = null;
String imgUrl = null;
String gender = null;
String user_type = null;
Integer userType = null;
Integer authType = null;
Boolean isDv = null;
TouTiaoAccount tta = null;
for (int i = 0; i < jsonArray.size(); i++) {
try {
JSONObject data = jsonArray.getJSONObject(i);
if(data.getLong("id") == null) {
if(data.getLong("media_id") == null) {
continue;
}
user_id = data.getLong("id");
name = data.getString("name");
if(data.getLong("media_id") != null) {
media_id = data.getLong("media_id");
}
description = data.getString("description");
if(data.getInteger("user_verified") != null) {
user_verified = data.getInteger("user_verified");
}
verify_content = data.getString("verify_content");
if(data.getInteger("follow_count") != null) {
follow_count = data.getInteger("follow_count");
}
img_url = "https:"+data.getString("avatar_url");
if(data.getString("create_time") != null) {
create_time = new Date(Long.valueOf(data.getString("create_time"))*1000);
}
gender = data.getString("gender");
user_type = data.getString("user_type");
if(name != null && name.equals(word)){
tta = new TouTiaoAccount(user_id, name, media_id, description, user_verified,
verify_content, follow_count,img_url,create_time, gender, user_type);
userId = data.getLong("id");
if(data.getLong("media_id") != null) {
mediaId = data.getLong("media_id");
}
description = data.getString("description");
verifyContent = data.getString("verify_content");
if(data.getJSONObject("user_auth_info") !=null && !data.getJSONObject("user_auth_info").isEmpty()) {
userVerified = 1;
verifyContent = data.getJSONObject("user_auth_info").getString("auth_info");
authType = data.getJSONObject("user_auth_info").getInteger("auth_type");
}
if(data.getInteger("follow_count") != null) {
followCount = data.getInteger("follow_count");
}
imgUrl = "https:"+data.getString("avatar_url");
if(data.getString("create_time") != null) {
createTime = new Date(Long.valueOf(data.getString("create_time"))*1000);
}
gender = data.getString("gender");
userType = data.getInteger("user_type");
tta = new TouTiaoAccount(userId, name, mediaId, description, userVerified,
verifyContent, followCount,imgUrl,createTime, gender, userType, authType, isDv);
ZhiWeiTools.sleep(1000);
TouTiaoAccount ttaUpdate = getTouTiaoAccountInfoByUserId(user_id+"", proxy);
TouTiaoAccount ttaUpdate = getTouTiaoAccountInfoByUserId(userId+"", proxy);
if(ttaUpdate != null){
tta.setFriend_count(ttaUpdate.getFriend_count());
tta.setUser_type(ttaUpdate.getUser_type());
tta.setFriendCount(ttaUpdate.getFriendCount());
tta.setUserType(ttaUpdate.getUserType());
tta.setIsDv(ttaUpdate.getIsDv());
}
return tta;
}
} catch (Exception e) {
logger.error("数据解析出现问题,{}", e);
continue;
}
}
} catch (Exception e) {
......@@ -307,18 +306,25 @@ public class TouTiaoAccountParse {
if(htmlBody.contains("var header={")){
String name = htmlBody.split("var header")[1].split("name:'")[1].split("',")[0];
String imgUrl = "https:"+htmlBody.split("avtar_img:'")[1].split("',")[0];
String type = htmlBody.split("type: '")[1].split("'")[0];
Integer type = Integer.valueOf(htmlBody.split("type: '")[1].split("'")[0]);
String isDvStr = htmlBody.split("dv:")[1].split(",")[0];
Boolean isDv = false;
if("true".equals(isDvStr)){
isDv = true;
}
if(htmlBody.contains("guanzhu")){
int guanzhu = Integer.valueOf(htmlBody.split("guanzhu:'")[1].split("',")[0]);
int fensi = Integer.valueOf(htmlBody.split("fensi:'")[1].split("',")[0]);
touTiaoAccount.setFriend_count(guanzhu);
touTiaoAccount.setFollow_count(fensi);
touTiaoAccount.setFriendCount(guanzhu);
touTiaoAccount.setFollowCount(fensi);
}
touTiaoAccount.setId(userId);
touTiaoAccount.setUser_id(Long.valueOf(userId));
touTiaoAccount.setImg_url(imgUrl);
touTiaoAccount.setUserId(Long.valueOf(userId));
touTiaoAccount.setImgUrl(imgUrl);
touTiaoAccount.setName(name);
touTiaoAccount.setUser_type(type);
touTiaoAccount.setUserType(type);
touTiaoAccount.setIsDv(isDv);
return touTiaoAccount;
}
} catch (Exception e) {
......@@ -345,54 +351,60 @@ public class TouTiaoAccountParse {
List<TouTiaoAccount> ttaList = new ArrayList<>();
try {
JSONArray jsonArray = json.getJSONArray("data");
long user_id = 0;
Long userId = null;
String name = null;
long media_id = 0;
Long mediaId = null;
String description = null;
int user_verified = 0;
String verify_content = null;
int follow_count = 0;
String img_url = null;
Date create_time = null;
Integer userVerified = 0;
String verifyContent = null;
Integer followCount = null;
Date createTime = null;
String imgUrl = null;
String gender = null;
String user_type = null;
Integer userType = null;
Integer authType = null;
Boolean isDv = null;
TouTiaoAccount tta = null;
for (int i = 0; i < jsonArray.size(); i++) {
try {
JSONObject data = jsonArray.getJSONObject(i);
user_id = data.getLong("id");
userId = data.getLong("id");
name = data.getString("name");
if(data.containsKey("media_id")) {
media_id = data.getLong("media_id");
mediaId = data.getLong("media_id");
}
description = data.getString("description");
user_verified = data.getInteger("user_verified");
verify_content = data.getString("verify_content");
follow_count = data.getInteger("follow_count");
img_url = "https:"+data.getString("avatar_url");
create_time = new Date(Long.parseLong((data.getString("create_time")+"000")));
verifyContent = data.getString("verify_content");
if(data.getJSONObject("user_auth_info") != null && !data.getJSONObject("user_auth_info").isEmpty()) {
userVerified = 1;
verifyContent = data.getJSONObject("user_auth_info").getString("auth_info");
authType = data.getJSONObject("user_auth_info").getInteger("auth_type");
}
followCount = data.getInteger("follow_count");
imgUrl = "https:"+data.getString("avatar_url");
createTime = new Date(Long.parseLong((data.getString("create_time")+"000")));
gender = data.getString("gender");
user_type = data.getString("user_type");
userType = data.getInteger("user_type");
tta = new TouTiaoAccount(user_id, name, media_id, description, user_verified,
verify_content, follow_count,img_url,create_time, gender, user_type);
tta = new TouTiaoAccount(userId, name, mediaId, description, userVerified,
verifyContent, followCount,imgUrl,createTime, gender, userType, authType, isDv);
if(Objects.nonNull(proxy)) {
ZhiWeiTools.sleep(1000);
}
TouTiaoAccount ttaUpdate = getTouTiaoAccountInfoByUserId(user_id+"", proxy);
TouTiaoAccount ttaUpdate = getTouTiaoAccountInfoByUserId(userId+"", proxy);
if(ttaUpdate != null){
tta.setFriend_count(ttaUpdate.getFriend_count());
tta.setUser_type(ttaUpdate.getUser_type());
tta.setFriendCount(ttaUpdate.getFriendCount());
tta.setUserType(ttaUpdate.getUserType());
tta.setIsDv(ttaUpdate.getIsDv());
}
ttaList.add(tta);
} catch (Exception e) {
logger.error("数据解析出现问题,{}", e);
continue;
}
}
} catch (Exception e) {
logger.error("数据解析出现问题,{}", e);
return null;
return Collections.emptyList();
}
return ttaList;
}
......@@ -411,39 +423,40 @@ public class TouTiaoAccountParse {
try {
ttaList = new ArrayList<>();
JSONArray jsonArray = json.getJSONArray("data");
Long user_id = null;
Long userId = null;
String name = null;
Long media_id = null;
Long mediaId = null;
String description = null;
Integer user_verified = null;
String verify_content = null;
int follow_count = 0;
String img_url = null;
Date create_time = null;
Integer userVerified = null;
String verifyContent = null;
Integer followCount = null;
Date createTime = null;
String imgUrl = null;
String gender = null;
String user_type = null;
Integer userType = null;
Integer authType = null;
Boolean isDv = null;
TouTiaoAccount tta = null;
for (int i = 0; i < jsonArray.size(); i++) {
try {
JSONObject data = jsonArray.getJSONObject(i);
user_id = data.getLong("user_id");
media_id = data.getLong("media_id");
userId = data.getLong("user_id");
mediaId = data.getLong("media_id");
name = data.getString("name");
img_url = "https:"+data.getString("avatar_url");
user_verified = data.getInteger("user_verified");
verify_content = data.getString("verified_content");
imgUrl = "https:"+data.getString("avatar_url");
userVerified = data.getInteger("user_verified");
verifyContent = data.getString("verified_content");
tta = new TouTiaoAccount(user_id, name, media_id, description, user_verified,
verify_content, follow_count, img_url, create_time, gender, user_type);
tta = new TouTiaoAccount(userId, name, mediaId, description, userVerified,
verifyContent, followCount, imgUrl, createTime, gender, userType, authType,isDv);
ttaList.add(tta);
} catch (Exception e) {
logger.error("数据解析出现问题,{}", e);
continue;
}
}
} catch (Exception e) {
logger.error("数据解析出现问题,{}", e);
return null;
return Collections.emptyList();
}
return ttaList;
}
......@@ -465,37 +478,38 @@ public class TouTiaoAccountParse {
try {
ttaList = new ArrayList<>();
JSONArray jsonArray = json.getJSONObject("data").getJSONArray("users");
Long user_id = null;
Long userId = null;
String name = null;
String description = null;
Integer user_verified = null;
String verify_content = null;
int follow_count = 0;
String img_url = null;
Date create_time = null;
Integer userVerified = -1;
String verifyContent = null;
Integer followCount = -1;
Date createTime = null;
String imgUrl = null;
String gender = null;
String user_type = null;
Integer userType = null;
Integer authType = null;
Boolean isDv = null;
TouTiaoAccount tta = null;
for (int i = 0; i < jsonArray.size(); i++) {
try {
JSONObject data = jsonArray.getJSONObject(i);
user_id = data.getLong("user_id");
userId = data.getLong("user_id");
name = data.getString("name");
description = data.getString("description");
verify_content = data.getString("verified_content");
img_url = data.getString("avatar_url");
user_verified = data.getBoolean("user_verified")==true? 0 : 1;
tta = new TouTiaoAccount(user_id, name, null, description, user_verified,
verify_content, follow_count, img_url, create_time, gender, user_type);
verifyContent = data.getString("verified_content");
imgUrl = data.getString("avatar_url");
userVerified = data.getBoolean("user_verified") == true? 0 : 1;
tta = new TouTiaoAccount(userId, name, null, description, userVerified,
verifyContent, followCount, imgUrl, createTime, gender, userType, authType, isDv);
ttaList.add(tta);
} catch (Exception e) {
logger.error("数据解析出现问题,{}", e);
continue;
}
}
} catch (Exception e) {
logger.error("数据解析出现问题,{}", e);
return null;
return Collections.emptyList();
}
return ttaList;
}
......@@ -516,12 +530,10 @@ public class TouTiaoAccountParse {
logger.error("获取数据时出现问题,问题为:{}", e.fillInStackTrace());
if(i==3){
break;
}else{
continue;
}
}
}
return null;
return Strings.EMPTY;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment