Skip to content

T
toutiao
Commit f061889a by zhiwei
Options

处理账号匹配规则

parent eedbd670
Showing with 46 additions and 51 deletions
src/main/java/com/zhiwei/toutiao/parse/TouTiaoAccountParse.java
......@@ -44,13 +44,13 @@ public class TouTiaoAccountParse {
try {
String htmlBody = null;
htmlBody = HttpClientTemplateOK.get(url, proxy, headerMap);
if(htmlBody != null && htmlBody.contains("media_id")){
if(htmlBody != null ){
tta = parseHtmlByAccount(htmlBody, name, proxy);
if(tta == null){
url = "https://www.toutiao.com/search_content/?offset=0&format=json&keyword="+URLCodeUtil.getURLEncode(name, "utf-8")+"&autoload=true&count=20&cur_tab=4&from=media";
headerMap.put("Referer","https://www.toutiao.com/search/?keyword="+URLCodeUtil.getURLEncode(name, "utf-8"));
htmlBody = HttpClientTemplateOK.get(url, proxy, headerMap);
if(htmlBody != null && htmlBody.contains("media_id")){
if(htmlBody != null){
tta = parseHtmlByAccount(htmlBody, name, proxy);
}
}
......@@ -58,7 +58,7 @@ public class TouTiaoAccountParse {
url = "https://www.toutiao.com/search_content/?offset=0&format=json&keyword="+URLCodeUtil.getURLEncode(name, "utf-8")+"&autoload=true&count=20&cur_tab=4&from=media";
headerMap.put("Referer","https://www.toutiao.com/search/?keyword="+URLCodeUtil.getURLEncode(name, "utf-8"));
htmlBody = HttpClientTemplateOK.get(url, proxy, headerMap);
if(htmlBody != null && htmlBody.contains("media_id")){
if(htmlBody != null){
tta = parseHtmlByAccount(htmlBody, name, proxy);
}
}
......@@ -79,7 +79,7 @@ public class TouTiaoAccountParse {
try {
String htmlBody = null;
htmlBody = HttpClientTemplateOK.get(url, proxy, headerMap);
if(htmlBody != null && htmlBody.contains("mediaId")){
if(htmlBody != null){
tta = parseAccountByUserId(htmlBody, user_id);
}
......@@ -113,7 +113,7 @@ public class TouTiaoAccountParse {
try {
String htmlBody = null;
htmlBody = HttpClientTemplateOK.get(url, proxy, headerMap);
if(htmlBody != null && htmlBody.contains("media_id")){
if(htmlBody != null){
JSONObject json = JSONObject.parseObject(htmlBody);
list.addAll(parseHtmlByWord(json, proxy));
if(json.getIntValue("has_more")==0){
......@@ -192,32 +192,29 @@ public class TouTiaoAccountParse {
JSONArray jsonArray = json.getJSONArray("data");
for (int i = 0; i < jsonArray.size(); i++) {
try {
JSONObject data = jsonArray.getJSONObject(i);
if(data.containsKey("media_id")){
long user_id = data.getLong("id");
String name = data.getString("name");
long media_id = data.getLong("media_id");
String description = data.getString("description");
int user_verified = data.getInteger("user_verified");
String verify_content = data.getString("verify_content");
int follow_count = data.getInteger("follow_count");
String img_url = "https:"+data.getString("avatar_url");
System.out.println(data.getString("create_time"));
Date create_time = new Date(Long.valueOf(data.getString("create_time"))*1000);
String gender = data.getString("gender");
String user_type = data.getString("user_type");
if(name.equals(word)){
TouTiaoAccount tta = new TouTiaoAccount(user_id, name, media_id, description, user_verified,
verify_content, follow_count,img_url,create_time, gender, user_type);
ZhiWeiTools.sleep(1000);
TouTiaoAccount ttaUpdate = getTouTiaoAccountInfoByUserId(user_id+"", proxy);
if(ttaUpdate != null){
tta.setFriend_count(ttaUpdate.getFriend_count());
tta.setUser_type(ttaUpdate.getUser_type());
}
return tta;
long user_id = data.getLong("id");
String name = data.getString("name");
long media_id = data.getLong("media_id");
String description = data.getString("description");
int user_verified = data.getInteger("user_verified");
String verify_content = data.getString("verify_content");
int follow_count = data.getInteger("follow_count");
String img_url = "https:"+data.getString("avatar_url");
System.out.println(data.getString("create_time"));
Date create_time = new Date(Long.valueOf(data.getString("create_time"))*1000);
String gender = data.getString("gender");
String user_type = data.getString("user_type");
if(name.equals(word)){
TouTiaoAccount tta = new TouTiaoAccount(user_id, name, media_id, description, user_verified,
verify_content, follow_count,img_url,create_time, gender, user_type);
ZhiWeiTools.sleep(1000);
TouTiaoAccount ttaUpdate = getTouTiaoAccountInfoByUserId(user_id+"", proxy);
if(ttaUpdate != null){
tta.setFriend_count(ttaUpdate.getFriend_count());
tta.setUser_type(ttaUpdate.getUser_type());
}
return tta;
}
} catch (Exception e) {
logger.error("数据解析出现问题,{}", e);
......@@ -299,29 +296,27 @@ public class TouTiaoAccountParse {
for (int i = 0; i < jsonArray.size(); i++) {
try {
JSONObject data = jsonArray.getJSONObject(i);
if(data.containsKey("media_id")){
user_id = data.getLong("id");
name = data.getString("name");
media_id = data.getLong("media_id");
description = data.getString("description");
user_verified = data.getInteger("user_verified");
verify_content = data.getString("verify_content");
follow_count = data.getInteger("follow_count");
img_url = "https:"+data.getString("avatar_url");
create_time = new Date(Integer.valueOf(data.getString("create_time")+"000"));
gender = data.getString("gender");
user_type = data.getString("user_type");
tta = new TouTiaoAccount(user_id, name, media_id, description, user_verified,
verify_content, follow_count,img_url,create_time, gender, user_type);
ZhiWeiTools.sleep(1000);
TouTiaoAccount ttaUpdate = getTouTiaoAccountInfoByUserId(user_id+"", proxy);
if(ttaUpdate != null){
tta.setFriend_count(ttaUpdate.getFriend_count());
tta.setUser_type(ttaUpdate.getUser_type());
}
ttaList.add(tta);
user_id = data.getLong("id");
name = data.getString("name");
media_id = data.getLong("media_id");
description = data.getString("description");
user_verified = data.getInteger("user_verified");
verify_content = data.getString("verify_content");
follow_count = data.getInteger("follow_count");
img_url = "https:"+data.getString("avatar_url");
create_time = new Date(Integer.valueOf(data.getString("create_time")+"000"));
gender = data.getString("gender");
user_type = data.getString("user_type");
tta = new TouTiaoAccount(user_id, name, media_id, description, user_verified,
verify_content, follow_count,img_url,create_time, gender, user_type);
ZhiWeiTools.sleep(1000);
TouTiaoAccount ttaUpdate = getTouTiaoAccountInfoByUserId(user_id+"", proxy);
if(ttaUpdate != null){
tta.setFriend_count(ttaUpdate.getFriend_count());
tta.setUser_type(ttaUpdate.getUser_type());
}
ttaList.add(tta);
} catch (Exception e) {
logger.error("数据解析出现问题,{}", e);
continue;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment